[ML] move find file structure finder in Rest high Level client to its new endpoint and plugin (#67290)

Find file structure finder is now its own plugin, and separated from the ml plugin.

This commit updates the rest high level client to reflect this.

Additionally, this adjusts the internal and client object names from `FileStructure` to the more general `TextStructure`
This commit is contained in:
Benjamin Trent 2021-01-14 08:16:52 -05:00 committed by GitHub
parent 4d03817dc2
commit 5cf569ffff
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
63 changed files with 945 additions and 768 deletions

View file

@ -45,6 +45,7 @@
<suppress files="client[/\\]rest-high-level[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]documentation[/\\]SnapshotClientDocumentationIT.java" id="SnippetLength" /> <suppress files="client[/\\]rest-high-level[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]documentation[/\\]SnapshotClientDocumentationIT.java" id="SnippetLength" />
<suppress files="client[/\\]rest-high-level[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]documentation[/\\]StoredScriptsDocumentationIT.java" id="SnippetLength" /> <suppress files="client[/\\]rest-high-level[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]documentation[/\\]StoredScriptsDocumentationIT.java" id="SnippetLength" />
<suppress files="client[/\\]rest-high-level[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]documentation[/\\]TasksClientDocumentationIT.java" id="SnippetLength" /> <suppress files="client[/\\]rest-high-level[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]documentation[/\\]TasksClientDocumentationIT.java" id="SnippetLength" />
<suppress files="client[/\\]rest-high-level[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]documentation[/\\]TextStructureClientDocumentationIT.java" id="SnippetLength" />
<suppress files="client[/\\]rest-high-level[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]documentation[/\\]WatcherDocumentationIT.java" id="SnippetLength" /> <suppress files="client[/\\]rest-high-level[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]documentation[/\\]WatcherDocumentationIT.java" id="SnippetLength" />
<suppress files="modules[/\\]reindex[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]documentation[/\\]ReindexDocumentationIT.java" id="SnippetLength" /> <suppress files="modules[/\\]reindex[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]documentation[/\\]ReindexDocumentationIT.java" id="SnippetLength" />

View file

@ -43,7 +43,6 @@ import org.elasticsearch.client.ml.DeleteTrainedModelRequest;
import org.elasticsearch.client.ml.EstimateModelMemoryRequest; import org.elasticsearch.client.ml.EstimateModelMemoryRequest;
import org.elasticsearch.client.ml.EvaluateDataFrameRequest; import org.elasticsearch.client.ml.EvaluateDataFrameRequest;
import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest; import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest;
import org.elasticsearch.client.ml.FindFileStructureRequest;
import org.elasticsearch.client.ml.FlushJobRequest; import org.elasticsearch.client.ml.FlushJobRequest;
import org.elasticsearch.client.ml.ForecastJobRequest; import org.elasticsearch.client.ml.ForecastJobRequest;
import org.elasticsearch.client.ml.GetBucketsRequest; import org.elasticsearch.client.ml.GetBucketsRequest;
@ -89,7 +88,6 @@ import org.elasticsearch.client.ml.UpdateModelSnapshotRequest;
import org.elasticsearch.client.ml.UpgradeJobModelSnapshotRequest; import org.elasticsearch.client.ml.UpgradeJobModelSnapshotRequest;
import org.elasticsearch.common.Strings; import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.XContentType;
import java.io.IOException; import java.io.IOException;
@ -939,63 +937,4 @@ final class MLRequestConverters {
return new Request(HttpGet.METHOD_NAME, endpoint); return new Request(HttpGet.METHOD_NAME, endpoint);
} }
static Request findFileStructure(FindFileStructureRequest findFileStructureRequest) {
String endpoint = new EndpointBuilder()
.addPathPartAsIs("_ml")
.addPathPartAsIs("find_file_structure")
.build();
Request request = new Request(HttpPost.METHOD_NAME, endpoint);
RequestConverters.Params params = new RequestConverters.Params();
if (findFileStructureRequest.getLinesToSample() != null) {
params.putParam(FindFileStructureRequest.LINES_TO_SAMPLE.getPreferredName(),
findFileStructureRequest.getLinesToSample().toString());
}
if (findFileStructureRequest.getTimeout() != null) {
params.putParam(FindFileStructureRequest.TIMEOUT.getPreferredName(), findFileStructureRequest.getTimeout().toString());
}
if (findFileStructureRequest.getCharset() != null) {
params.putParam(FindFileStructureRequest.CHARSET.getPreferredName(), findFileStructureRequest.getCharset());
}
if (findFileStructureRequest.getFormat() != null) {
params.putParam(FindFileStructureRequest.FORMAT.getPreferredName(), findFileStructureRequest.getFormat().toString());
}
if (findFileStructureRequest.getColumnNames() != null) {
params.putParam(FindFileStructureRequest.COLUMN_NAMES.getPreferredName(),
Strings.collectionToCommaDelimitedString(findFileStructureRequest.getColumnNames()));
}
if (findFileStructureRequest.getHasHeaderRow() != null) {
params.putParam(FindFileStructureRequest.HAS_HEADER_ROW.getPreferredName(),
findFileStructureRequest.getHasHeaderRow().toString());
}
if (findFileStructureRequest.getDelimiter() != null) {
params.putParam(FindFileStructureRequest.DELIMITER.getPreferredName(),
findFileStructureRequest.getDelimiter().toString());
}
if (findFileStructureRequest.getQuote() != null) {
params.putParam(FindFileStructureRequest.QUOTE.getPreferredName(), findFileStructureRequest.getQuote().toString());
}
if (findFileStructureRequest.getShouldTrimFields() != null) {
params.putParam(FindFileStructureRequest.SHOULD_TRIM_FIELDS.getPreferredName(),
findFileStructureRequest.getShouldTrimFields().toString());
}
if (findFileStructureRequest.getGrokPattern() != null) {
params.putParam(FindFileStructureRequest.GROK_PATTERN.getPreferredName(), findFileStructureRequest.getGrokPattern());
}
if (findFileStructureRequest.getTimestampFormat() != null) {
params.putParam(FindFileStructureRequest.TIMESTAMP_FORMAT.getPreferredName(), findFileStructureRequest.getTimestampFormat());
}
if (findFileStructureRequest.getTimestampField() != null) {
params.putParam(FindFileStructureRequest.TIMESTAMP_FIELD.getPreferredName(), findFileStructureRequest.getTimestampField());
}
if (findFileStructureRequest.getExplain() != null) {
params.putParam(FindFileStructureRequest.EXPLAIN.getPreferredName(), findFileStructureRequest.getExplain().toString());
}
request.addParameters(params.asMap());
BytesReference sample = findFileStructureRequest.getSample();
BytesRef source = sample.toBytesRef();
HttpEntity byteEntity = new NByteArrayEntity(source.bytes, source.offset, source.length, createContentType(XContentType.JSON));
request.setEntity(byteEntity);
return request;
}
} }

View file

@ -41,8 +41,6 @@ import org.elasticsearch.client.ml.DeleteJobResponse;
import org.elasticsearch.client.ml.DeleteModelSnapshotRequest; import org.elasticsearch.client.ml.DeleteModelSnapshotRequest;
import org.elasticsearch.client.ml.EvaluateDataFrameRequest; import org.elasticsearch.client.ml.EvaluateDataFrameRequest;
import org.elasticsearch.client.ml.EvaluateDataFrameResponse; import org.elasticsearch.client.ml.EvaluateDataFrameResponse;
import org.elasticsearch.client.ml.FindFileStructureRequest;
import org.elasticsearch.client.ml.FindFileStructureResponse;
import org.elasticsearch.client.ml.FlushJobRequest; import org.elasticsearch.client.ml.FlushJobRequest;
import org.elasticsearch.client.ml.FlushJobResponse; import org.elasticsearch.client.ml.FlushJobResponse;
import org.elasticsearch.client.ml.ForecastJobRequest; import org.elasticsearch.client.ml.ForecastJobRequest;
@ -1919,48 +1917,6 @@ public final class MachineLearningClient {
Collections.emptySet()); Collections.emptySet());
} }
/**
* Finds the structure of a file
* <p>
* For additional info
* see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-find-file-structure.html">
* ML Find File Structure documentation</a>
*
* @param request The find file structure request
* @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @return the response containing details of the file structure
* @throws IOException when there is a serialization issue sending the request or receiving the response
*/
public FindFileStructureResponse findFileStructure(FindFileStructureRequest request, RequestOptions options) throws IOException {
return restHighLevelClient.performRequestAndParseEntity(request,
MLRequestConverters::findFileStructure,
options,
FindFileStructureResponse::fromXContent,
Collections.emptySet());
}
/**
* Finds the structure of a file asynchronously and notifies the listener on completion
* <p>
* For additional info
* see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-find-file-structure.html">
* ML Find File Structure documentation</a>
*
* @param request The find file structure request
* @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @param listener Listener to be notified upon request completion
* @return cancellable that may be used to cancel the request
*/
public Cancellable findFileStructureAsync(FindFileStructureRequest request, RequestOptions options,
ActionListener<FindFileStructureResponse> listener) {
return restHighLevelClient.performRequestAsyncAndParseEntity(request,
MLRequestConverters::findFileStructure,
options,
FindFileStructureResponse::fromXContent,
listener,
Collections.emptySet());
}
/** /**
* Sets the ML cluster setting upgrade_mode * Sets the ML cluster setting upgrade_mode
* <p> * <p>

View file

@ -274,6 +274,7 @@ public class RestHighLevelClient implements Closeable {
private final EnrichClient enrichClient = new EnrichClient(this); private final EnrichClient enrichClient = new EnrichClient(this);
private final EqlClient eqlClient = new EqlClient(this); private final EqlClient eqlClient = new EqlClient(this);
private final AsyncSearchClient asyncSearchClient = new AsyncSearchClient(this); private final AsyncSearchClient asyncSearchClient = new AsyncSearchClient(this);
private final TextStructureClient textStructureClient = new TextStructureClient(this);
/** /**
* Creates a {@link RestHighLevelClient} given the low level {@link RestClientBuilder} that allows to build the * Creates a {@link RestHighLevelClient} given the low level {@link RestClientBuilder} that allows to build the
@ -454,6 +455,16 @@ public class RestHighLevelClient implements Closeable {
return asyncSearchClient; return asyncSearchClient;
} }
/**
* A wrapper for the {@link RestHighLevelClient} that provides methods for accessing the Elastic Text Structure APIs.
* <p>
* See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/find-structure.html"> X-Pack APIs on elastic.co</a>
* for more information.
*/
public TextStructureClient textStructure() {
return textStructureClient;
}
/** /**
* Provides methods for accessing the Elastic Licensed Migration APIs that * Provides methods for accessing the Elastic Licensed Migration APIs that
* are shipped with the default distribution of Elasticsearch. All of * are shipped with the default distribution of Elasticsearch. All of

View file

@ -0,0 +1,85 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client;
import java.io.IOException;
import java.util.Collections;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.client.textstructure.FindStructureRequest;
import org.elasticsearch.client.textstructure.FindStructureResponse;
/**
* Text Structure API client wrapper for the {@link RestHighLevelClient}
* <p>
* See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/find-structure.html">
* X-Pack Text Structure APIs </a> for additional information.
*/
public final class TextStructureClient {
private final RestHighLevelClient restHighLevelClient;
TextStructureClient(RestHighLevelClient restHighLevelClient) {
this.restHighLevelClient = restHighLevelClient;
}
/**
* Finds the structure of a file
* <p>
* For additional info
* see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/find-structure.html">
* Text Structure Find Structure documentation</a>
*
* @param request The find file structure request
* @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @return the response containing details of the file structure
* @throws IOException when there is a serialization issue sending the request or receiving the response
*/
public FindStructureResponse findStructure(FindStructureRequest request, RequestOptions options) throws IOException {
return restHighLevelClient.performRequestAndParseEntity(request,
TextStructureRequestConverters::findFileStructure,
options,
FindStructureResponse::fromXContent,
Collections.emptySet());
}
/**
* Finds the structure of a file asynchronously and notifies the listener on completion
* <p>
* For additional info
* see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/find-structure.html">
* Text Structure Find Structure documentation</a>
*
* @param request The find file structure request
* @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @param listener Listener to be notified upon request completion
* @return cancellable that may be used to cancel the request
*/
public Cancellable findStructureAsync(FindStructureRequest request, RequestOptions options,
ActionListener<FindStructureResponse> listener) {
return restHighLevelClient.performRequestAsyncAndParseEntity(request,
TextStructureRequestConverters::findFileStructure,
options,
FindStructureResponse::fromXContent,
listener,
Collections.emptySet());
}
}

View file

@ -0,0 +1,97 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client;
import static org.elasticsearch.client.RequestConverters.createContentType;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.nio.entity.NByteArrayEntity;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.client.RequestConverters.EndpointBuilder;
import org.elasticsearch.client.textstructure.FindStructureRequest;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.XContentType;
final class TextStructureRequestConverters {
private TextStructureRequestConverters() {}
static Request findFileStructure(FindStructureRequest findStructureRequest) {
String endpoint = new EndpointBuilder()
.addPathPartAsIs("_text_structure")
.addPathPartAsIs("find_structure")
.build();
Request request = new Request(HttpPost.METHOD_NAME, endpoint);
RequestConverters.Params params = new RequestConverters.Params();
if (findStructureRequest.getLinesToSample() != null) {
params.putParam(FindStructureRequest.LINES_TO_SAMPLE.getPreferredName(),
findStructureRequest.getLinesToSample().toString());
}
if (findStructureRequest.getTimeout() != null) {
params.putParam(FindStructureRequest.TIMEOUT.getPreferredName(), findStructureRequest.getTimeout().toString());
}
if (findStructureRequest.getCharset() != null) {
params.putParam(FindStructureRequest.CHARSET.getPreferredName(), findStructureRequest.getCharset());
}
if (findStructureRequest.getFormat() != null) {
params.putParam(FindStructureRequest.FORMAT.getPreferredName(), findStructureRequest.getFormat().toString());
}
if (findStructureRequest.getColumnNames() != null) {
params.putParam(FindStructureRequest.COLUMN_NAMES.getPreferredName(),
Strings.collectionToCommaDelimitedString(findStructureRequest.getColumnNames()));
}
if (findStructureRequest.getHasHeaderRow() != null) {
params.putParam(FindStructureRequest.HAS_HEADER_ROW.getPreferredName(),
findStructureRequest.getHasHeaderRow().toString());
}
if (findStructureRequest.getDelimiter() != null) {
params.putParam(FindStructureRequest.DELIMITER.getPreferredName(),
findStructureRequest.getDelimiter().toString());
}
if (findStructureRequest.getQuote() != null) {
params.putParam(FindStructureRequest.QUOTE.getPreferredName(), findStructureRequest.getQuote().toString());
}
if (findStructureRequest.getShouldTrimFields() != null) {
params.putParam(FindStructureRequest.SHOULD_TRIM_FIELDS.getPreferredName(),
findStructureRequest.getShouldTrimFields().toString());
}
if (findStructureRequest.getGrokPattern() != null) {
params.putParam(FindStructureRequest.GROK_PATTERN.getPreferredName(), findStructureRequest.getGrokPattern());
}
if (findStructureRequest.getTimestampFormat() != null) {
params.putParam(FindStructureRequest.TIMESTAMP_FORMAT.getPreferredName(), findStructureRequest.getTimestampFormat());
}
if (findStructureRequest.getTimestampField() != null) {
params.putParam(FindStructureRequest.TIMESTAMP_FIELD.getPreferredName(), findStructureRequest.getTimestampField());
}
if (findStructureRequest.getExplain() != null) {
params.putParam(FindStructureRequest.EXPLAIN.getPreferredName(), findStructureRequest.getExplain().toString());
}
request.addParameters(params.asMap());
BytesReference sample = findStructureRequest.getSample();
BytesRef source = sample.toBytesRef();
HttpEntity byteEntity = new NByteArrayEntity(source.bytes, source.offset, source.length, createContentType(XContentType.JSON));
request.setEntity(byteEntity);
return request;
}
}

View file

@ -16,11 +16,11 @@
* specific language governing permissions and limitations * specific language governing permissions and limitations
* under the License. * under the License.
*/ */
package org.elasticsearch.client.ml; package org.elasticsearch.client.textstructure;
import org.elasticsearch.client.Validatable; import org.elasticsearch.client.Validatable;
import org.elasticsearch.client.ValidationException; import org.elasticsearch.client.ValidationException;
import org.elasticsearch.client.ml.filestructurefinder.FileStructure; import org.elasticsearch.client.textstructure.structurefinder.TextStructure;
import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.BytesReference;
@ -34,29 +34,29 @@ import java.util.List;
import java.util.Objects; import java.util.Objects;
import java.util.Optional; import java.util.Optional;
public class FindFileStructureRequest implements Validatable, ToXContentFragment { public class FindStructureRequest implements Validatable, ToXContentFragment {
public static final ParseField LINES_TO_SAMPLE = new ParseField("lines_to_sample"); public static final ParseField LINES_TO_SAMPLE = new ParseField("lines_to_sample");
public static final ParseField LINE_MERGE_SIZE_LIMIT = new ParseField("line_merge_size_limit"); public static final ParseField LINE_MERGE_SIZE_LIMIT = new ParseField("line_merge_size_limit");
public static final ParseField TIMEOUT = new ParseField("timeout"); public static final ParseField TIMEOUT = new ParseField("timeout");
public static final ParseField CHARSET = FileStructure.CHARSET; public static final ParseField CHARSET = TextStructure.CHARSET;
public static final ParseField FORMAT = FileStructure.FORMAT; public static final ParseField FORMAT = TextStructure.FORMAT;
public static final ParseField COLUMN_NAMES = FileStructure.COLUMN_NAMES; public static final ParseField COLUMN_NAMES = TextStructure.COLUMN_NAMES;
public static final ParseField HAS_HEADER_ROW = FileStructure.HAS_HEADER_ROW; public static final ParseField HAS_HEADER_ROW = TextStructure.HAS_HEADER_ROW;
public static final ParseField DELIMITER = FileStructure.DELIMITER; public static final ParseField DELIMITER = TextStructure.DELIMITER;
public static final ParseField QUOTE = FileStructure.QUOTE; public static final ParseField QUOTE = TextStructure.QUOTE;
public static final ParseField SHOULD_TRIM_FIELDS = FileStructure.SHOULD_TRIM_FIELDS; public static final ParseField SHOULD_TRIM_FIELDS = TextStructure.SHOULD_TRIM_FIELDS;
public static final ParseField GROK_PATTERN = FileStructure.GROK_PATTERN; public static final ParseField GROK_PATTERN = TextStructure.GROK_PATTERN;
// This one is plural in FileStructure, but singular in FileStructureOverrides // This one is plural in FileStructure, but singular in FileStructureOverrides
public static final ParseField TIMESTAMP_FORMAT = new ParseField("timestamp_format"); public static final ParseField TIMESTAMP_FORMAT = new ParseField("timestamp_format");
public static final ParseField TIMESTAMP_FIELD = FileStructure.TIMESTAMP_FIELD; public static final ParseField TIMESTAMP_FIELD = TextStructure.TIMESTAMP_FIELD;
public static final ParseField EXPLAIN = new ParseField("explain"); public static final ParseField EXPLAIN = new ParseField("explain");
private Integer linesToSample; private Integer linesToSample;
private Integer lineMergeSizeLimit; private Integer lineMergeSizeLimit;
private TimeValue timeout; private TimeValue timeout;
private String charset; private String charset;
private FileStructure.Format format; private TextStructure.Format format;
private List<String> columnNames; private List<String> columnNames;
private Boolean hasHeaderRow; private Boolean hasHeaderRow;
private Character delimiter; private Character delimiter;
@ -68,7 +68,7 @@ public class FindFileStructureRequest implements Validatable, ToXContentFragment
private Boolean explain; private Boolean explain;
private BytesReference sample; private BytesReference sample;
public FindFileStructureRequest() { public FindStructureRequest() {
} }
public Integer getLinesToSample() { public Integer getLinesToSample() {
@ -103,16 +103,16 @@ public class FindFileStructureRequest implements Validatable, ToXContentFragment
this.charset = (charset == null || charset.isEmpty()) ? null : charset; this.charset = (charset == null || charset.isEmpty()) ? null : charset;
} }
public FileStructure.Format getFormat() { public TextStructure.Format getFormat() {
return format; return format;
} }
public void setFormat(FileStructure.Format format) { public void setFormat(TextStructure.Format format) {
this.format = format; this.format = format;
} }
public void setFormat(String format) { public void setFormat(String format) {
this.format = (format == null || format.isEmpty()) ? null : FileStructure.Format.fromString(format); this.format = (format == null || format.isEmpty()) ? null : TextStructure.Format.fromString(format);
} }
public List<String> getColumnNames() { public List<String> getColumnNames() {
@ -298,7 +298,7 @@ public class FindFileStructureRequest implements Validatable, ToXContentFragment
return false; return false;
} }
FindFileStructureRequest that = (FindFileStructureRequest) other; FindStructureRequest that = (FindStructureRequest) other;
return Objects.equals(this.linesToSample, that.linesToSample) && return Objects.equals(this.linesToSample, that.linesToSample) &&
Objects.equals(this.lineMergeSizeLimit, that.lineMergeSizeLimit) && Objects.equals(this.lineMergeSizeLimit, that.lineMergeSizeLimit) &&
Objects.equals(this.timeout, that.timeout) && Objects.equals(this.timeout, that.timeout) &&

View file

@ -16,9 +16,9 @@
* specific language governing permissions and limitations * specific language governing permissions and limitations
* under the License. * under the License.
*/ */
package org.elasticsearch.client.ml; package org.elasticsearch.client.textstructure;
import org.elasticsearch.client.ml.filestructurefinder.FileStructure; import org.elasticsearch.client.textstructure.structurefinder.TextStructure;
import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser;
@ -26,31 +26,31 @@ import org.elasticsearch.common.xcontent.XContentParser;
import java.io.IOException; import java.io.IOException;
import java.util.Objects; import java.util.Objects;
public class FindFileStructureResponse implements ToXContentObject { public class FindStructureResponse implements ToXContentObject {
private final FileStructure fileStructure; private final TextStructure textStructure;
FindFileStructureResponse(FileStructure fileStructure) { FindStructureResponse(TextStructure textStructure) {
this.fileStructure = Objects.requireNonNull(fileStructure); this.textStructure = Objects.requireNonNull(textStructure);
} }
public static FindFileStructureResponse fromXContent(XContentParser parser) throws IOException { public static FindStructureResponse fromXContent(XContentParser parser) throws IOException {
return new FindFileStructureResponse(FileStructure.PARSER.parse(parser, null).build()); return new FindStructureResponse(TextStructure.PARSER.parse(parser, null).build());
} }
public FileStructure getFileStructure() { public TextStructure getFileStructure() {
return fileStructure; return textStructure;
} }
@Override @Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
fileStructure.toXContent(builder, params); textStructure.toXContent(builder, params);
return builder; return builder;
} }
@Override @Override
public int hashCode() { public int hashCode() {
return Objects.hash(fileStructure); return Objects.hash(textStructure);
} }
@Override @Override
@ -64,7 +64,7 @@ public class FindFileStructureResponse implements ToXContentObject {
return false; return false;
} }
FindFileStructureResponse that = (FindFileStructureResponse) other; FindStructureResponse that = (FindStructureResponse) other;
return Objects.equals(fileStructure, that.fileStructure); return Objects.equals(textStructure, that.textStructure);
} }
} }

View file

@ -16,7 +16,7 @@
* specific language governing permissions and limitations * specific language governing permissions and limitations
* under the License. * under the License.
*/ */
package org.elasticsearch.client.ml.filestructurefinder; package org.elasticsearch.client.textstructure.structurefinder;
import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ConstructingObjectParser;

View file

@ -16,7 +16,7 @@
* specific language governing permissions and limitations * specific language governing permissions and limitations
* under the License. * under the License.
*/ */
package org.elasticsearch.client.ml.filestructurefinder; package org.elasticsearch.client.textstructure.structurefinder;
import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ObjectParser;
@ -35,9 +35,9 @@ import java.util.SortedMap;
import java.util.TreeMap; import java.util.TreeMap;
/** /**
* Stores the file format determined by Machine Learning. * Stores the determined text format.
*/ */
public class FileStructure implements ToXContentObject { public class TextStructure implements ToXContentObject {
public enum Format { public enum Format {
@ -132,7 +132,7 @@ public class FileStructure implements ToXContentObject {
private final SortedMap<String, FieldStats> fieldStats; private final SortedMap<String, FieldStats> fieldStats;
private final List<String> explanation; private final List<String> explanation;
private FileStructure(int numLinesAnalyzed, int numMessagesAnalyzed, String sampleStart, String charset, Boolean hasByteOrderMarker, private TextStructure(int numLinesAnalyzed, int numMessagesAnalyzed, String sampleStart, String charset, Boolean hasByteOrderMarker,
Format format, String multilineStartPattern, String excludeLinesPattern, List<String> columnNames, Format format, String multilineStartPattern, String excludeLinesPattern, List<String> columnNames,
Boolean hasHeaderRow, Character delimiter, Character quote, Boolean shouldTrimFields, String grokPattern, Boolean hasHeaderRow, Character delimiter, Character quote, Boolean shouldTrimFields, String grokPattern,
String timestampField, List<String> jodaTimestampFormats, List<String> javaTimestampFormats, String timestampField, List<String> jodaTimestampFormats, List<String> javaTimestampFormats,
@ -335,7 +335,7 @@ public class FileStructure implements ToXContentObject {
return false; return false;
} }
FileStructure that = (FileStructure) other; TextStructure that = (TextStructure) other;
return this.numLinesAnalyzed == that.numLinesAnalyzed && return this.numLinesAnalyzed == that.numLinesAnalyzed &&
this.numMessagesAnalyzed == that.numMessagesAnalyzed && this.numMessagesAnalyzed == that.numMessagesAnalyzed &&
Objects.equals(this.sampleStart, that.sampleStart) && Objects.equals(this.sampleStart, that.sampleStart) &&
@ -502,9 +502,9 @@ public class FileStructure implements ToXContentObject {
return this; return this;
} }
public FileStructure build() { public TextStructure build() {
return new FileStructure(numLinesAnalyzed, numMessagesAnalyzed, sampleStart, charset, hasByteOrderMarker, format, return new TextStructure(numLinesAnalyzed, numMessagesAnalyzed, sampleStart, charset, hasByteOrderMarker, format,
multilineStartPattern, excludeLinesPattern, columnNames, hasHeaderRow, delimiter, quote, shouldTrimFields, grokPattern, multilineStartPattern, excludeLinesPattern, columnNames, hasHeaderRow, delimiter, quote, shouldTrimFields, grokPattern,
timestampField, jodaTimestampFormats, javaTimestampFormats, needClientTimezone, mappings, ingestPipeline, fieldStats, timestampField, jodaTimestampFormats, javaTimestampFormats, needClientTimezone, mappings, ingestPipeline, fieldStats,
explanation); explanation);

View file

@ -40,8 +40,6 @@ import org.elasticsearch.client.ml.EstimateModelMemoryRequest;
import org.elasticsearch.client.ml.EvaluateDataFrameRequest; import org.elasticsearch.client.ml.EvaluateDataFrameRequest;
import org.elasticsearch.client.ml.EvaluateDataFrameRequestTests; import org.elasticsearch.client.ml.EvaluateDataFrameRequestTests;
import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest; import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest;
import org.elasticsearch.client.ml.FindFileStructureRequest;
import org.elasticsearch.client.ml.FindFileStructureRequestTests;
import org.elasticsearch.client.ml.FlushJobRequest; import org.elasticsearch.client.ml.FlushJobRequest;
import org.elasticsearch.client.ml.ForecastJobRequest; import org.elasticsearch.client.ml.ForecastJobRequest;
import org.elasticsearch.client.ml.GetBucketsRequest; import org.elasticsearch.client.ml.GetBucketsRequest;
@ -96,7 +94,6 @@ import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsConfigUpdate;
import org.elasticsearch.client.ml.dataframe.MlDataFrameAnalysisNamedXContentProvider; import org.elasticsearch.client.ml.dataframe.MlDataFrameAnalysisNamedXContentProvider;
import org.elasticsearch.client.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider; import org.elasticsearch.client.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider;
import org.elasticsearch.client.ml.dataframe.stats.AnalysisStatsNamedXContentProvider; import org.elasticsearch.client.ml.dataframe.stats.AnalysisStatsNamedXContentProvider;
import org.elasticsearch.client.ml.filestructurefinder.FileStructure;
import org.elasticsearch.client.ml.inference.MlInferenceNamedXContentProvider; import org.elasticsearch.client.ml.inference.MlInferenceNamedXContentProvider;
import org.elasticsearch.client.ml.inference.TrainedModelConfig; import org.elasticsearch.client.ml.inference.TrainedModelConfig;
import org.elasticsearch.client.ml.inference.TrainedModelConfigTests; import org.elasticsearch.client.ml.inference.TrainedModelConfigTests;
@ -121,7 +118,6 @@ import org.elasticsearch.test.ESTestCase;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
@ -1051,85 +1047,6 @@ public class MLRequestConvertersTests extends ESTestCase {
assertNull(request.getEntity()); assertNull(request.getEntity());
} }
public void testFindFileStructure() throws Exception {
String sample = randomAlphaOfLength(randomIntBetween(1000, 2000));
FindFileStructureRequest findFileStructureRequest = FindFileStructureRequestTests.createTestRequestWithoutSample();
findFileStructureRequest.setSample(sample.getBytes(StandardCharsets.UTF_8));
Request request = MLRequestConverters.findFileStructure(findFileStructureRequest);
assertEquals(HttpPost.METHOD_NAME, request.getMethod());
assertEquals("/_ml/find_file_structure", request.getEndpoint());
if (findFileStructureRequest.getLinesToSample() != null) {
assertEquals(findFileStructureRequest.getLinesToSample(), Integer.valueOf(request.getParameters().get("lines_to_sample")));
} else {
assertNull(request.getParameters().get("lines_to_sample"));
}
if (findFileStructureRequest.getTimeout() != null) {
assertEquals(findFileStructureRequest.getTimeout().toString(), request.getParameters().get("timeout"));
} else {
assertNull(request.getParameters().get("timeout"));
}
if (findFileStructureRequest.getCharset() != null) {
assertEquals(findFileStructureRequest.getCharset(), request.getParameters().get("charset"));
} else {
assertNull(request.getParameters().get("charset"));
}
if (findFileStructureRequest.getFormat() != null) {
assertEquals(findFileStructureRequest.getFormat(), FileStructure.Format.fromString(request.getParameters().get("format")));
} else {
assertNull(request.getParameters().get("format"));
}
if (findFileStructureRequest.getColumnNames() != null) {
assertEquals(findFileStructureRequest.getColumnNames(),
Arrays.asList(Strings.splitStringByCommaToArray(request.getParameters().get("column_names"))));
} else {
assertNull(request.getParameters().get("column_names"));
}
if (findFileStructureRequest.getHasHeaderRow() != null) {
assertEquals(findFileStructureRequest.getHasHeaderRow(), Boolean.valueOf(request.getParameters().get("has_header_row")));
} else {
assertNull(request.getParameters().get("has_header_row"));
}
if (findFileStructureRequest.getDelimiter() != null) {
assertEquals(findFileStructureRequest.getDelimiter().toString(), request.getParameters().get("delimiter"));
} else {
assertNull(request.getParameters().get("delimiter"));
}
if (findFileStructureRequest.getQuote() != null) {
assertEquals(findFileStructureRequest.getQuote().toString(), request.getParameters().get("quote"));
} else {
assertNull(request.getParameters().get("quote"));
}
if (findFileStructureRequest.getShouldTrimFields() != null) {
assertEquals(findFileStructureRequest.getShouldTrimFields(),
Boolean.valueOf(request.getParameters().get("should_trim_fields")));
} else {
assertNull(request.getParameters().get("should_trim_fields"));
}
if (findFileStructureRequest.getGrokPattern() != null) {
assertEquals(findFileStructureRequest.getGrokPattern(), request.getParameters().get("grok_pattern"));
} else {
assertNull(request.getParameters().get("grok_pattern"));
}
if (findFileStructureRequest.getTimestampFormat() != null) {
assertEquals(findFileStructureRequest.getTimestampFormat(), request.getParameters().get("timestamp_format"));
} else {
assertNull(request.getParameters().get("timestamp_format"));
}
if (findFileStructureRequest.getTimestampField() != null) {
assertEquals(findFileStructureRequest.getTimestampField(), request.getParameters().get("timestamp_field"));
} else {
assertNull(request.getParameters().get("timestamp_field"));
}
if (findFileStructureRequest.getExplain() != null) {
assertEquals(findFileStructureRequest.getExplain(), Boolean.valueOf(request.getParameters().get("explain")));
} else {
assertNull(request.getParameters().get("explain"));
}
assertEquals(sample, requestEntityToString(request));
}
public void testSetUpgradeMode() { public void testSetUpgradeMode() {
SetUpgradeModeRequest setUpgradeModeRequest = new SetUpgradeModeRequest(true); SetUpgradeModeRequest setUpgradeModeRequest = new SetUpgradeModeRequest(true);

View file

@ -54,8 +54,6 @@ import org.elasticsearch.client.ml.EvaluateDataFrameRequest;
import org.elasticsearch.client.ml.EvaluateDataFrameResponse; import org.elasticsearch.client.ml.EvaluateDataFrameResponse;
import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest; import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest;
import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsResponse; import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsResponse;
import org.elasticsearch.client.ml.FindFileStructureRequest;
import org.elasticsearch.client.ml.FindFileStructureResponse;
import org.elasticsearch.client.ml.FlushJobRequest; import org.elasticsearch.client.ml.FlushJobRequest;
import org.elasticsearch.client.ml.FlushJobResponse; import org.elasticsearch.client.ml.FlushJobResponse;
import org.elasticsearch.client.ml.ForecastJobRequest; import org.elasticsearch.client.ml.ForecastJobRequest;
@ -160,7 +158,6 @@ import org.elasticsearch.client.ml.dataframe.explain.FieldSelection;
import org.elasticsearch.client.ml.dataframe.explain.MemoryEstimation; import org.elasticsearch.client.ml.dataframe.explain.MemoryEstimation;
import org.elasticsearch.client.ml.dataframe.stats.common.DataCounts; import org.elasticsearch.client.ml.dataframe.stats.common.DataCounts;
import org.elasticsearch.client.ml.dataframe.stats.common.MemoryUsage; import org.elasticsearch.client.ml.dataframe.stats.common.MemoryUsage;
import org.elasticsearch.client.ml.filestructurefinder.FileStructure;
import org.elasticsearch.client.ml.inference.InferenceToXContentCompressor; import org.elasticsearch.client.ml.inference.InferenceToXContentCompressor;
import org.elasticsearch.client.ml.inference.MlInferenceNamedXContentProvider; import org.elasticsearch.client.ml.inference.MlInferenceNamedXContentProvider;
import org.elasticsearch.client.ml.inference.TrainedModelConfig; import org.elasticsearch.client.ml.inference.TrainedModelConfig;
@ -205,7 +202,6 @@ import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@ -2881,54 +2877,6 @@ public class MachineLearningIT extends ESRestHighLevelClientTestCase {
} }
} }
public void testFindFileStructure() throws IOException {
String sample = "{\"logger\":\"controller\",\"timestamp\":1478261151445,\"level\":\"INFO\"," +
"\"pid\":42,\"thread\":\"0x7fff7d2a8000\",\"message\":\"message 1\",\"class\":\"ml\"," +
"\"method\":\"core::SomeNoiseMaker\",\"file\":\"Noisemaker.cc\",\"line\":333}\n" +
"{\"logger\":\"controller\",\"timestamp\":1478261151445," +
"\"level\":\"INFO\",\"pid\":42,\"thread\":\"0x7fff7d2a8000\",\"message\":\"message 2\",\"class\":\"ml\"," +
"\"method\":\"core::SomeNoiseMaker\",\"file\":\"Noisemaker.cc\",\"line\":333}\n";
MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
FindFileStructureRequest request = new FindFileStructureRequest();
request.setSample(sample.getBytes(StandardCharsets.UTF_8));
FindFileStructureResponse response = execute(
request,
machineLearningClient::findFileStructure,
machineLearningClient::findFileStructureAsync,
RequestOptions.DEFAULT
.toBuilder()
.setWarningsHandler(
warnings -> Collections.singletonList(
"[POST /_ml/find_file_structure] is deprecated! Use [POST /_text_structure/find_structure] instead."
).equals(warnings) == false
).build());
FileStructure structure = response.getFileStructure();
assertEquals(2, structure.getNumLinesAnalyzed());
assertEquals(2, structure.getNumMessagesAnalyzed());
assertEquals(sample, structure.getSampleStart());
assertEquals(FileStructure.Format.NDJSON, structure.getFormat());
assertEquals(StandardCharsets.UTF_8.displayName(Locale.ROOT), structure.getCharset());
assertFalse(structure.getHasByteOrderMarker());
assertNull(structure.getMultilineStartPattern());
assertNull(structure.getExcludeLinesPattern());
assertNull(structure.getColumnNames());
assertNull(structure.getHasHeaderRow());
assertNull(structure.getDelimiter());
assertNull(structure.getQuote());
assertNull(structure.getShouldTrimFields());
assertNull(structure.getGrokPattern());
assertEquals(Collections.singletonList("UNIX_MS"), structure.getJavaTimestampFormats());
assertEquals(Collections.singletonList("UNIX_MS"), structure.getJodaTimestampFormats());
assertEquals("timestamp", structure.getTimestampField());
assertFalse(structure.needClientTimezone());
}
public void testEnableUpgradeMode() throws Exception { public void testEnableUpgradeMode() throws Exception {
MachineLearningClient machineLearningClient = highLevelClient().machineLearning(); MachineLearningClient machineLearningClient = highLevelClient().machineLearning();

View file

@ -915,6 +915,7 @@ public class RestHighLevelClientTests extends ESTestCase {
apiName.startsWith("ccr.") == false && apiName.startsWith("ccr.") == false &&
apiName.startsWith("enrich.") == false && apiName.startsWith("enrich.") == false &&
apiName.startsWith("transform.") == false && apiName.startsWith("transform.") == false &&
apiName.startsWith("text_structure.") == false &&
apiName.startsWith("eql.") == false && apiName.startsWith("eql.") == false &&
apiName.endsWith("freeze") == false && apiName.endsWith("freeze") == false &&
apiName.endsWith("reload_analyzers") == false && apiName.endsWith("reload_analyzers") == false &&

View file

@ -0,0 +1,74 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.Locale;
import org.elasticsearch.client.textstructure.FindStructureRequest;
import org.elasticsearch.client.textstructure.FindStructureResponse;
import org.elasticsearch.client.textstructure.structurefinder.TextStructure;
public class TextStructureIT extends ESRestHighLevelClientTestCase {
public void testFindFileStructure() throws IOException {
String sample = "{\"logger\":\"controller\",\"timestamp\":1478261151445,\"level\":\"INFO\"," +
"\"pid\":42,\"thread\":\"0x7fff7d2a8000\",\"message\":\"message 1\",\"class\":\"ml\"," +
"\"method\":\"core::SomeNoiseMaker\",\"file\":\"Noisemaker.cc\",\"line\":333}\n" +
"{\"logger\":\"controller\",\"timestamp\":1478261151445," +
"\"level\":\"INFO\",\"pid\":42,\"thread\":\"0x7fff7d2a8000\",\"message\":\"message 2\",\"class\":\"ml\"," +
"\"method\":\"core::SomeNoiseMaker\",\"file\":\"Noisemaker.cc\",\"line\":333}\n";
TextStructureClient textStructureClient = highLevelClient().textStructure();
FindStructureRequest request = new FindStructureRequest();
request.setSample(sample.getBytes(StandardCharsets.UTF_8));
FindStructureResponse response = execute(
request,
textStructureClient::findStructure,
textStructureClient::findStructureAsync,
RequestOptions.DEFAULT);
TextStructure structure = response.getFileStructure();
assertEquals(2, structure.getNumLinesAnalyzed());
assertEquals(2, structure.getNumMessagesAnalyzed());
assertEquals(sample, structure.getSampleStart());
assertEquals(TextStructure.Format.NDJSON, structure.getFormat());
assertEquals(StandardCharsets.UTF_8.displayName(Locale.ROOT), structure.getCharset());
assertFalse(structure.getHasByteOrderMarker());
assertNull(structure.getMultilineStartPattern());
assertNull(structure.getExcludeLinesPattern());
assertNull(structure.getColumnNames());
assertNull(structure.getHasHeaderRow());
assertNull(structure.getDelimiter());
assertNull(structure.getQuote());
assertNull(structure.getShouldTrimFields());
assertNull(structure.getGrokPattern());
assertEquals(Collections.singletonList("UNIX_MS"), structure.getJavaTimestampFormats());
assertEquals(Collections.singletonList("UNIX_MS"), structure.getJodaTimestampFormats());
assertEquals("timestamp", structure.getTimestampField());
assertFalse(structure.needClientTimezone());
}
}

View file

@ -0,0 +1,119 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client;
import java.io.ByteArrayOutputStream;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import org.apache.http.client.methods.HttpPost;
import org.elasticsearch.client.textstructure.FindStructureRequest;
import org.elasticsearch.client.textstructure.FindStructureRequestTests;
import org.elasticsearch.client.textstructure.structurefinder.TextStructure;
import org.elasticsearch.common.Strings;
import org.elasticsearch.test.ESTestCase;
public class TextStructureRequestConvertersTests extends ESTestCase {
public void testFindFileStructure() throws Exception {
String sample = randomAlphaOfLength(randomIntBetween(1000, 2000));
FindStructureRequest findStructureRequest = FindStructureRequestTests.createTestRequestWithoutSample();
findStructureRequest.setSample(sample.getBytes(StandardCharsets.UTF_8));
Request request = TextStructureRequestConverters.findFileStructure(findStructureRequest);
assertEquals(HttpPost.METHOD_NAME, request.getMethod());
assertEquals("/_text_structure/find_structure", request.getEndpoint());
if (findStructureRequest.getLinesToSample() != null) {
assertEquals(findStructureRequest.getLinesToSample(), Integer.valueOf(request.getParameters().get("lines_to_sample")));
} else {
assertNull(request.getParameters().get("lines_to_sample"));
}
if (findStructureRequest.getTimeout() != null) {
assertEquals(findStructureRequest.getTimeout().toString(), request.getParameters().get("timeout"));
} else {
assertNull(request.getParameters().get("timeout"));
}
if (findStructureRequest.getCharset() != null) {
assertEquals(findStructureRequest.getCharset(), request.getParameters().get("charset"));
} else {
assertNull(request.getParameters().get("charset"));
}
if (findStructureRequest.getFormat() != null) {
assertEquals(findStructureRequest.getFormat(), TextStructure.Format.fromString(request.getParameters().get("format")));
} else {
assertNull(request.getParameters().get("format"));
}
if (findStructureRequest.getColumnNames() != null) {
assertEquals(findStructureRequest.getColumnNames(),
Arrays.asList(Strings.splitStringByCommaToArray(request.getParameters().get("column_names"))));
} else {
assertNull(request.getParameters().get("column_names"));
}
if (findStructureRequest.getHasHeaderRow() != null) {
assertEquals(findStructureRequest.getHasHeaderRow(), Boolean.valueOf(request.getParameters().get("has_header_row")));
} else {
assertNull(request.getParameters().get("has_header_row"));
}
if (findStructureRequest.getDelimiter() != null) {
assertEquals(findStructureRequest.getDelimiter().toString(), request.getParameters().get("delimiter"));
} else {
assertNull(request.getParameters().get("delimiter"));
}
if (findStructureRequest.getQuote() != null) {
assertEquals(findStructureRequest.getQuote().toString(), request.getParameters().get("quote"));
} else {
assertNull(request.getParameters().get("quote"));
}
if (findStructureRequest.getShouldTrimFields() != null) {
assertEquals(findStructureRequest.getShouldTrimFields(),
Boolean.valueOf(request.getParameters().get("should_trim_fields")));
} else {
assertNull(request.getParameters().get("should_trim_fields"));
}
if (findStructureRequest.getGrokPattern() != null) {
assertEquals(findStructureRequest.getGrokPattern(), request.getParameters().get("grok_pattern"));
} else {
assertNull(request.getParameters().get("grok_pattern"));
}
if (findStructureRequest.getTimestampFormat() != null) {
assertEquals(findStructureRequest.getTimestampFormat(), request.getParameters().get("timestamp_format"));
} else {
assertNull(request.getParameters().get("timestamp_format"));
}
if (findStructureRequest.getTimestampField() != null) {
assertEquals(findStructureRequest.getTimestampField(), request.getParameters().get("timestamp_field"));
} else {
assertNull(request.getParameters().get("timestamp_field"));
}
if (findStructureRequest.getExplain() != null) {
assertEquals(findStructureRequest.getExplain(), Boolean.valueOf(request.getParameters().get("explain")));
} else {
assertNull(request.getParameters().get("explain"));
}
assertEquals(sample, requestEntityToString(request));
}
private static String requestEntityToString(Request request) throws Exception {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
request.getEntity().writeTo(bos);
return bos.toString("UTF-8");
}
}

View file

@ -56,8 +56,6 @@ import org.elasticsearch.client.ml.EvaluateDataFrameRequest;
import org.elasticsearch.client.ml.EvaluateDataFrameResponse; import org.elasticsearch.client.ml.EvaluateDataFrameResponse;
import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest; import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest;
import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsResponse; import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsResponse;
import org.elasticsearch.client.ml.FindFileStructureRequest;
import org.elasticsearch.client.ml.FindFileStructureResponse;
import org.elasticsearch.client.ml.FlushJobRequest; import org.elasticsearch.client.ml.FlushJobRequest;
import org.elasticsearch.client.ml.FlushJobResponse; import org.elasticsearch.client.ml.FlushJobResponse;
import org.elasticsearch.client.ml.ForecastJobRequest; import org.elasticsearch.client.ml.ForecastJobRequest;
@ -175,7 +173,6 @@ import org.elasticsearch.client.ml.dataframe.evaluation.regression.MeanSquaredLo
import org.elasticsearch.client.ml.dataframe.evaluation.regression.RSquaredMetric; import org.elasticsearch.client.ml.dataframe.evaluation.regression.RSquaredMetric;
import org.elasticsearch.client.ml.dataframe.explain.FieldSelection; import org.elasticsearch.client.ml.dataframe.explain.FieldSelection;
import org.elasticsearch.client.ml.dataframe.explain.MemoryEstimation; import org.elasticsearch.client.ml.dataframe.explain.MemoryEstimation;
import org.elasticsearch.client.ml.filestructurefinder.FileStructure;
import org.elasticsearch.client.ml.inference.InferenceToXContentCompressor; import org.elasticsearch.client.ml.inference.InferenceToXContentCompressor;
import org.elasticsearch.client.ml.inference.MlInferenceNamedXContentProvider; import org.elasticsearch.client.ml.inference.MlInferenceNamedXContentProvider;
import org.elasticsearch.client.ml.inference.TrainedModelConfig; import org.elasticsearch.client.ml.inference.TrainedModelConfig;
@ -221,9 +218,6 @@ import org.elasticsearch.tasks.TaskId;
import org.junit.After; import org.junit.After;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.Date; import java.util.Date;
@ -1846,77 +1840,6 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase {
} }
} }
public void testFindFileStructure() throws Exception {
RestHighLevelClient client = highLevelClient();
Path anInterestingFile = createTempFile();
String contents = "{\"logger\":\"controller\",\"timestamp\":1478261151445,\"level\":\"INFO\"," +
"\"pid\":42,\"thread\":\"0x7fff7d2a8000\",\"message\":\"message 1\",\"class\":\"ml\"," +
"\"method\":\"core::SomeNoiseMaker\",\"file\":\"Noisemaker.cc\",\"line\":333}\n" +
"{\"logger\":\"controller\",\"timestamp\":1478261151445," +
"\"level\":\"INFO\",\"pid\":42,\"thread\":\"0x7fff7d2a8000\",\"message\":\"message 2\",\"class\":\"ml\"," +
"\"method\":\"core::SomeNoiseMaker\",\"file\":\"Noisemaker.cc\",\"line\":333}\n";
Files.write(anInterestingFile, Collections.singleton(contents), StandardCharsets.UTF_8);
{
// tag::find-file-structure-request
FindFileStructureRequest findFileStructureRequest = new FindFileStructureRequest(); // <1>
findFileStructureRequest.setSample(Files.readAllBytes(anInterestingFile)); // <2>
// end::find-file-structure-request
// tag::find-file-structure-request-options
findFileStructureRequest.setLinesToSample(500); // <1>
findFileStructureRequest.setExplain(true); // <2>
// end::find-file-structure-request-options
// tag::find-file-structure-execute
FindFileStructureResponse findFileStructureResponse = client
.machineLearning()
.findFileStructure(
findFileStructureRequest,
RequestOptions.DEFAULT
.toBuilder()
.setWarningsHandler(
warnings -> Collections.singletonList(
"[POST /_ml/find_file_structure] is deprecated! Use [POST /_text_structure/find_structure] instead."
).equals(warnings) == false
).build());
// end::find-file-structure-execute
// tag::find-file-structure-response
FileStructure structure = findFileStructureResponse.getFileStructure(); // <1>
// end::find-file-structure-response
assertEquals(2, structure.getNumLinesAnalyzed());
}
{
// tag::find-file-structure-execute-listener
ActionListener<FindFileStructureResponse> listener = new ActionListener<FindFileStructureResponse>() {
@Override
public void onResponse(FindFileStructureResponse findFileStructureResponse) {
// <1>
}
@Override
public void onFailure(Exception e) {
// <2>
}
};
// end::find-file-structure-execute-listener
FindFileStructureRequest findFileStructureRequest = new FindFileStructureRequest();
findFileStructureRequest.setSample(Files.readAllBytes(anInterestingFile));
// Replace the empty listener by a blocking listener in test
final CountDownLatch latch = new CountDownLatch(1);
listener = new LatchedActionListener<>(listener, latch);
// tag::find-file-structure-execute-async
client.machineLearning().findFileStructureAsync(findFileStructureRequest, RequestOptions.DEFAULT, listener); // <1>
// end::find-file-structure-execute-async
assertTrue(latch.await(30L, TimeUnit.SECONDS));
}
}
public void testGetInfluencers() throws IOException, InterruptedException { public void testGetInfluencers() throws IOException, InterruptedException {
RestHighLevelClient client = highLevelClient(); RestHighLevelClient client = highLevelClient();

View file

@ -0,0 +1,107 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.documentation;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.LatchedActionListener;
import org.elasticsearch.client.ESRestHighLevelClientTestCase;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.textstructure.FindStructureRequest;
import org.elasticsearch.client.textstructure.FindStructureResponse;
import org.elasticsearch.client.textstructure.structurefinder.TextStructure;
public class TextStructureClientDocumentationIT extends ESRestHighLevelClientTestCase {
public void testFindStructure() throws Exception {
RestHighLevelClient client = highLevelClient();
Path anInterestingFile = createTempFile();
String contents = "{\"logger\":\"controller\",\"timestamp\":1478261151445,\"level\":\"INFO\"," +
"\"pid\":42,\"thread\":\"0x7fff7d2a8000\",\"message\":\"message 1\",\"class\":\"ml\"," +
"\"method\":\"core::SomeNoiseMaker\",\"file\":\"Noisemaker.cc\",\"line\":333}\n" +
"{\"logger\":\"controller\",\"timestamp\":1478261151445," +
"\"level\":\"INFO\",\"pid\":42,\"thread\":\"0x7fff7d2a8000\",\"message\":\"message 2\",\"class\":\"ml\"," +
"\"method\":\"core::SomeNoiseMaker\",\"file\":\"Noisemaker.cc\",\"line\":333}\n";
Files.write(anInterestingFile, Collections.singleton(contents), StandardCharsets.UTF_8);
{
// tag::find-structure-request
FindStructureRequest request = new FindStructureRequest(); // <1>
request.setSample(Files.readAllBytes(anInterestingFile)); // <2>
// end::find-structure-request
// tag::find-structure-request-options
request.setLinesToSample(500); // <1>
request.setExplain(true); // <2>
// end::find-structure-request-options
// tag::find-structure-execute
FindStructureResponse response = client
.textStructure()
.findStructure(
request,
RequestOptions.DEFAULT
);
// end::find-structure-execute
// tag::find-structure-response
TextStructure structure = response.getFileStructure(); // <1>
// end::find-structure-response
assertEquals(2, structure.getNumLinesAnalyzed());
}
{
// tag::find-structure-execute-listener
ActionListener<FindStructureResponse> listener = new ActionListener<>() {
@Override
public void onResponse(FindStructureResponse response) {
// <1>
}
@Override
public void onFailure(Exception e) {
// <2>
}
};
// end::find-structure-execute-listener
FindStructureRequest request = new FindStructureRequest();
request.setSample(Files.readAllBytes(anInterestingFile));
// Replace the empty listener by a blocking listener in test
final CountDownLatch latch = new CountDownLatch(1);
listener = new LatchedActionListener<>(listener, latch);
// tag::find-structure-execute-async
client
.textStructure()
.findStructureAsync(request, RequestOptions.DEFAULT, listener); // <1>
// end::find-structure-execute-async
assertTrue(latch.await(30L, TimeUnit.SECONDS));
}
}
}

View file

@ -1,118 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.ml;
import org.elasticsearch.client.ml.filestructurefinder.FileStructure;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.test.AbstractXContentTestCase;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.Arrays;
public class FindFileStructureRequestTests extends AbstractXContentTestCase<FindFileStructureRequest> {
private static final ObjectParser<FindFileStructureRequest, Void> PARSER =
new ObjectParser<>("find_file_structure_request", FindFileStructureRequest::new);
static {
PARSER.declareInt(FindFileStructureRequest::setLinesToSample, FindFileStructureRequest.LINES_TO_SAMPLE);
PARSER.declareInt(FindFileStructureRequest::setLineMergeSizeLimit, FindFileStructureRequest.LINE_MERGE_SIZE_LIMIT);
PARSER.declareString((p, c) -> p.setTimeout(TimeValue.parseTimeValue(c, FindFileStructureRequest.TIMEOUT.getPreferredName())),
FindFileStructureRequest.TIMEOUT);
PARSER.declareString(FindFileStructureRequest::setCharset, FindFileStructureRequest.CHARSET);
PARSER.declareString(FindFileStructureRequest::setFormat, FindFileStructureRequest.FORMAT);
PARSER.declareStringArray(FindFileStructureRequest::setColumnNames, FindFileStructureRequest.COLUMN_NAMES);
PARSER.declareBoolean(FindFileStructureRequest::setHasHeaderRow, FindFileStructureRequest.HAS_HEADER_ROW);
PARSER.declareString(FindFileStructureRequest::setDelimiter, FindFileStructureRequest.DELIMITER);
PARSER.declareString(FindFileStructureRequest::setQuote, FindFileStructureRequest.QUOTE);
PARSER.declareBoolean(FindFileStructureRequest::setShouldTrimFields, FindFileStructureRequest.SHOULD_TRIM_FIELDS);
PARSER.declareString(FindFileStructureRequest::setGrokPattern, FindFileStructureRequest.GROK_PATTERN);
PARSER.declareString(FindFileStructureRequest::setTimestampFormat, FindFileStructureRequest.TIMESTAMP_FORMAT);
PARSER.declareString(FindFileStructureRequest::setTimestampField, FindFileStructureRequest.TIMESTAMP_FIELD);
PARSER.declareBoolean(FindFileStructureRequest::setExplain, FindFileStructureRequest.EXPLAIN);
// Sample is not included in the X-Content representation
}
@Override
protected FindFileStructureRequest doParseInstance(XContentParser parser) throws IOException {
return PARSER.apply(parser, null);
}
@Override
protected boolean supportsUnknownFields() {
return false;
}
@Override
protected FindFileStructureRequest createTestInstance() {
return createTestRequestWithoutSample();
}
public static FindFileStructureRequest createTestRequestWithoutSample() {
FindFileStructureRequest findFileStructureRequest = new FindFileStructureRequest();
if (randomBoolean()) {
findFileStructureRequest.setLinesToSample(randomIntBetween(1000, 2000));
}
if (randomBoolean()) {
findFileStructureRequest.setLineMergeSizeLimit(randomIntBetween(10000, 20000));
}
if (randomBoolean()) {
findFileStructureRequest.setTimeout(TimeValue.timeValueSeconds(randomIntBetween(10, 20)));
}
if (randomBoolean()) {
findFileStructureRequest.setCharset(Charset.defaultCharset().toString());
}
if (randomBoolean()) {
findFileStructureRequest.setFormat(randomFrom(FileStructure.Format.values()));
}
if (randomBoolean()) {
findFileStructureRequest.setColumnNames(Arrays.asList(generateRandomStringArray(10, 10, false, false)));
}
if (randomBoolean()) {
findFileStructureRequest.setHasHeaderRow(randomBoolean());
}
if (randomBoolean()) {
findFileStructureRequest.setDelimiter(randomAlphaOfLength(1));
}
if (randomBoolean()) {
findFileStructureRequest.setQuote(randomAlphaOfLength(1));
}
if (randomBoolean()) {
findFileStructureRequest.setShouldTrimFields(randomBoolean());
}
if (randomBoolean()) {
findFileStructureRequest.setGrokPattern(randomAlphaOfLength(100));
}
if (randomBoolean()) {
findFileStructureRequest.setTimestampFormat(randomAlphaOfLength(10));
}
if (randomBoolean()) {
findFileStructureRequest.setTimestampField(randomAlphaOfLength(10));
}
if (randomBoolean()) {
findFileStructureRequest.setExplain(randomBoolean());
}
return findFileStructureRequest;
}
}

View file

@ -0,0 +1,118 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.textstructure;
import org.elasticsearch.client.textstructure.structurefinder.TextStructure;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.test.AbstractXContentTestCase;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.Arrays;
public class FindStructureRequestTests extends AbstractXContentTestCase<FindStructureRequest> {
private static final ObjectParser<FindStructureRequest, Void> PARSER =
new ObjectParser<>("find_file_structure_request", FindStructureRequest::new);
static {
PARSER.declareInt(FindStructureRequest::setLinesToSample, FindStructureRequest.LINES_TO_SAMPLE);
PARSER.declareInt(FindStructureRequest::setLineMergeSizeLimit, FindStructureRequest.LINE_MERGE_SIZE_LIMIT);
PARSER.declareString((p, c) -> p.setTimeout(TimeValue.parseTimeValue(c, FindStructureRequest.TIMEOUT.getPreferredName())),
FindStructureRequest.TIMEOUT);
PARSER.declareString(FindStructureRequest::setCharset, FindStructureRequest.CHARSET);
PARSER.declareString(FindStructureRequest::setFormat, FindStructureRequest.FORMAT);
PARSER.declareStringArray(FindStructureRequest::setColumnNames, FindStructureRequest.COLUMN_NAMES);
PARSER.declareBoolean(FindStructureRequest::setHasHeaderRow, FindStructureRequest.HAS_HEADER_ROW);
PARSER.declareString(FindStructureRequest::setDelimiter, FindStructureRequest.DELIMITER);
PARSER.declareString(FindStructureRequest::setQuote, FindStructureRequest.QUOTE);
PARSER.declareBoolean(FindStructureRequest::setShouldTrimFields, FindStructureRequest.SHOULD_TRIM_FIELDS);
PARSER.declareString(FindStructureRequest::setGrokPattern, FindStructureRequest.GROK_PATTERN);
PARSER.declareString(FindStructureRequest::setTimestampFormat, FindStructureRequest.TIMESTAMP_FORMAT);
PARSER.declareString(FindStructureRequest::setTimestampField, FindStructureRequest.TIMESTAMP_FIELD);
PARSER.declareBoolean(FindStructureRequest::setExplain, FindStructureRequest.EXPLAIN);
// Sample is not included in the X-Content representation
}
@Override
protected FindStructureRequest doParseInstance(XContentParser parser) throws IOException {
return PARSER.apply(parser, null);
}
@Override
protected boolean supportsUnknownFields() {
return false;
}
@Override
protected FindStructureRequest createTestInstance() {
return createTestRequestWithoutSample();
}
public static FindStructureRequest createTestRequestWithoutSample() {
FindStructureRequest findStructureRequest = new FindStructureRequest();
if (randomBoolean()) {
findStructureRequest.setLinesToSample(randomIntBetween(1000, 2000));
}
if (randomBoolean()) {
findStructureRequest.setLineMergeSizeLimit(randomIntBetween(10000, 20000));
}
if (randomBoolean()) {
findStructureRequest.setTimeout(TimeValue.timeValueSeconds(randomIntBetween(10, 20)));
}
if (randomBoolean()) {
findStructureRequest.setCharset(Charset.defaultCharset().toString());
}
if (randomBoolean()) {
findStructureRequest.setFormat(randomFrom(TextStructure.Format.values()));
}
if (randomBoolean()) {
findStructureRequest.setColumnNames(Arrays.asList(generateRandomStringArray(10, 10, false, false)));
}
if (randomBoolean()) {
findStructureRequest.setHasHeaderRow(randomBoolean());
}
if (randomBoolean()) {
findStructureRequest.setDelimiter(randomAlphaOfLength(1));
}
if (randomBoolean()) {
findStructureRequest.setQuote(randomAlphaOfLength(1));
}
if (randomBoolean()) {
findStructureRequest.setShouldTrimFields(randomBoolean());
}
if (randomBoolean()) {
findStructureRequest.setGrokPattern(randomAlphaOfLength(100));
}
if (randomBoolean()) {
findStructureRequest.setTimestampFormat(randomAlphaOfLength(10));
}
if (randomBoolean()) {
findStructureRequest.setTimestampField(randomAlphaOfLength(10));
}
if (randomBoolean()) {
findStructureRequest.setExplain(randomBoolean());
}
return findStructureRequest;
}
}

View file

@ -16,25 +16,25 @@
* specific language governing permissions and limitations * specific language governing permissions and limitations
* under the License. * under the License.
*/ */
package org.elasticsearch.client.ml; package org.elasticsearch.client.textstructure;
import org.elasticsearch.client.ml.filestructurefinder.FileStructureTests; import org.elasticsearch.client.textstructure.structurefinder.TextStructureTests;
import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.test.AbstractXContentTestCase; import org.elasticsearch.test.AbstractXContentTestCase;
import java.io.IOException; import java.io.IOException;
import java.util.function.Predicate; import java.util.function.Predicate;
public class FindFileStructureResponseTests extends AbstractXContentTestCase<FindFileStructureResponse> { public class FindStructureResponseTests extends AbstractXContentTestCase<FindStructureResponse> {
@Override @Override
protected FindFileStructureResponse createTestInstance() { protected FindStructureResponse createTestInstance() {
return new FindFileStructureResponse(FileStructureTests.createTestFileStructure()); return new FindStructureResponse(TextStructureTests.createTestFileStructure());
} }
@Override @Override
protected FindFileStructureResponse doParseInstance(XContentParser parser) throws IOException { protected FindStructureResponse doParseInstance(XContentParser parser) throws IOException {
return FindFileStructureResponse.fromXContent(parser); return FindStructureResponse.fromXContent(parser);
} }
@Override @Override

View file

@ -16,8 +16,9 @@
* specific language governing permissions and limitations * specific language governing permissions and limitations
* under the License. * under the License.
*/ */
package org.elasticsearch.client.ml.filestructurefinder; package org.elasticsearch.client.textstructure.structurefinder;
import org.elasticsearch.client.textstructure.structurefinder.FieldStats;
import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.test.AbstractXContentTestCase; import org.elasticsearch.test.AbstractXContentTestCase;

View file

@ -16,7 +16,7 @@
* specific language governing permissions and limitations * specific language governing permissions and limitations
* under the License. * under the License.
*/ */
package org.elasticsearch.client.ml.filestructurefinder; package org.elasticsearch.client.textstructure.structurefinder;
import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.test.AbstractXContentTestCase; import org.elasticsearch.test.AbstractXContentTestCase;
@ -31,18 +31,18 @@ import java.util.Map;
import java.util.TreeMap; import java.util.TreeMap;
import java.util.function.Predicate; import java.util.function.Predicate;
public class FileStructureTests extends AbstractXContentTestCase<FileStructure> { public class TextStructureTests extends AbstractXContentTestCase<TextStructure> {
@Override @Override
protected FileStructure createTestInstance() { protected TextStructure createTestInstance() {
return createTestFileStructure(); return createTestFileStructure();
} }
public static FileStructure createTestFileStructure() { public static TextStructure createTestFileStructure() {
FileStructure.Format format = randomFrom(EnumSet.allOf(FileStructure.Format.class)); TextStructure.Format format = randomFrom(EnumSet.allOf(TextStructure.Format.class));
FileStructure.Builder builder = new FileStructure.Builder(format); TextStructure.Builder builder = new TextStructure.Builder(format);
int numLinesAnalyzed = randomIntBetween(2, 10000); int numLinesAnalyzed = randomIntBetween(2, 10000);
builder.setNumLinesAnalyzed(numLinesAnalyzed); builder.setNumLinesAnalyzed(numLinesAnalyzed);
@ -63,18 +63,18 @@ public class FileStructureTests extends AbstractXContentTestCase<FileStructure>
builder.setExcludeLinesPattern(randomAlphaOfLength(100)); builder.setExcludeLinesPattern(randomAlphaOfLength(100));
} }
if (format == FileStructure.Format.DELIMITED) { if (format == TextStructure.Format.DELIMITED) {
builder.setColumnNames(Arrays.asList(generateRandomStringArray(10, 10, false, false))); builder.setColumnNames(Arrays.asList(generateRandomStringArray(10, 10, false, false)));
builder.setHasHeaderRow(randomBoolean()); builder.setHasHeaderRow(randomBoolean());
builder.setDelimiter(randomFrom(',', '\t', ';', '|')); builder.setDelimiter(randomFrom(',', '\t', ';', '|'));
builder.setQuote(randomFrom('"', '\'')); builder.setQuote(randomFrom('"', '\''));
} }
if (format == FileStructure.Format.SEMI_STRUCTURED_TEXT) { if (format == TextStructure.Format.SEMI_STRUCTURED_TEXT) {
builder.setGrokPattern(randomAlphaOfLength(100)); builder.setGrokPattern(randomAlphaOfLength(100));
} }
if (format == FileStructure.Format.SEMI_STRUCTURED_TEXT || randomBoolean()) { if (format == TextStructure.Format.SEMI_STRUCTURED_TEXT || randomBoolean()) {
builder.setTimestampField(randomAlphaOfLength(10)); builder.setTimestampField(randomAlphaOfLength(10));
builder.setJodaTimestampFormats(Arrays.asList(generateRandomStringArray(3, 20, false, false))); builder.setJodaTimestampFormats(Arrays.asList(generateRandomStringArray(3, 20, false, false)));
builder.setJavaTimestampFormats(Arrays.asList(generateRandomStringArray(3, 20, false, false))); builder.setJavaTimestampFormats(Arrays.asList(generateRandomStringArray(3, 20, false, false)));
@ -109,8 +109,8 @@ public class FileStructureTests extends AbstractXContentTestCase<FileStructure>
} }
@Override @Override
protected FileStructure doParseInstance(XContentParser parser) { protected TextStructure doParseInstance(XContentParser parser) {
return FileStructure.PARSER.apply(parser, null).build(); return TextStructure.PARSER.apply(parser, null).build();
} }
@Override @Override

View file

@ -301,7 +301,6 @@ The Java High Level REST Client supports the following {ml} APIs:
* <<{upid}-estimate-model-memory>> * <<{upid}-estimate-model-memory>>
* <<{upid}-evaluate-data-frame>> * <<{upid}-evaluate-data-frame>>
* <<{upid}-explain-data-frame-analytics>> * <<{upid}-explain-data-frame-analytics>>
* <<{upid}-find-file-structure>>
* <<{upid}-flush-job>> * <<{upid}-flush-job>>
* <<{upid}-forecast-job>> * <<{upid}-forecast-job>>
* <<{upid}-get-job>> * <<{upid}-get-job>>
@ -366,8 +365,6 @@ include::ml/estimate-model-memory.asciidoc[]
include::ml/evaluate-data-frame.asciidoc[] include::ml/evaluate-data-frame.asciidoc[]
// EXPLAIN // EXPLAIN
include::ml/explain-data-frame-analytics.asciidoc[] include::ml/explain-data-frame-analytics.asciidoc[]
// FIND
include::ml/find-file-structure.asciidoc[]
// FLUSH // FLUSH
include::ml/flush-job.asciidoc[] include::ml/flush-job.asciidoc[]
// FORECAST // FORECAST
@ -529,6 +526,17 @@ include::security/create-api-key.asciidoc[]
include::security/get-api-key.asciidoc[] include::security/get-api-key.asciidoc[]
include::security/invalidate-api-key.asciidoc[] include::security/invalidate-api-key.asciidoc[]
[role="xpack"]
== Text Structure APIs
:upid: {mainid}-text-structure
:doc-tests-file: {doc-tests}/TextStructureClientDocumentationIT.java
The Java High Level REST Client supports the following Text Structure APIs:
* <<{upid}-find-structure>>
include::textstructure/find-structure.asciidoc[]
[role="xpack"] [role="xpack"]
== Watcher APIs == Watcher APIs

View file

@ -1,30 +1,29 @@
-- --
:api: find-file-structure :api: find-structure
:request: FindFileStructureRequest :request: FindStructureRequest
:response: FindFileStructureResponse :response: FindStructureResponse
-- --
[role="xpack"] [role="xpack"]
[id="{upid}-{api}"] [id="{upid}-{api}"]
=== Find file structure API === Find structure API
experimental::[] experimental::[]
Determines the structure of a text file and other information that will be Determines the structure of text and other information that will be
useful to import its contents to an {es} index. It accepts a +{request}+ object useful to import its contents to an {es} index. It accepts a +{request}+ object
and responds with a +{response}+ object. and responds with a +{response}+ object.
[id="{upid}-{api}-request"] [id="{upid}-{api}-request"]
==== Find file structure request ==== Find structure request
A sample from the beginning of the file (or the entire file contents if A sample of the text to analyze must be added to the +{request}+
it's small) must be added to the +{request}+ object using the object using the `FindStructureRequest#setSample` method.
`FindFileStructureRequest#setSample` method.
["source","java",subs="attributes,callouts,macros"] ["source","java",subs="attributes,callouts,macros"]
-------------------------------------------------- --------------------------------------------------
include-tagged::{doc-tests-file}[{api}-request] include-tagged::{doc-tests-file}[{api}-request]
-------------------------------------------------- --------------------------------------------------
<1> Create a new `FindFileStructureRequest` object <1> Create a new `FindStructureRequest` object
<2> Add the contents of `anInterestingFile` to the request <2> Add the contents of `anInterestingFile` to the request
==== Optional arguments ==== Optional arguments
@ -42,9 +41,9 @@ include-tagged::{doc-tests-file}[{api}-request-options]
include::../execution.asciidoc[] include::../execution.asciidoc[]
[id="{upid}-{api}-response"] [id="{upid}-{api}-response"]
==== Find file structure response ==== Find structure response
A +{response}+ contains information about the file structure, A +{response}+ contains information about the text structure,
as well as mappings and an ingest pipeline that could be used as well as mappings and an ingest pipeline that could be used
to index the contents into {es}. to index the contents into {es}.
@ -52,4 +51,4 @@ to index the contents into {es}.
-------------------------------------------------- --------------------------------------------------
include-tagged::{doc-tests-file}[{api}-response] include-tagged::{doc-tests-file}[{api}-response]
-------------------------------------------------- --------------------------------------------------
<1> The `FileStructure` object contains the structure information <1> The `TextStructure` object contains the structure information

View file

@ -90,7 +90,7 @@ import org.elasticsearch.xpack.core.ml.action.DeleteTrainedModelAction;
import org.elasticsearch.xpack.core.ml.action.EvaluateDataFrameAction; import org.elasticsearch.xpack.core.ml.action.EvaluateDataFrameAction;
import org.elasticsearch.xpack.core.ml.action.ExplainDataFrameAnalyticsAction; import org.elasticsearch.xpack.core.ml.action.ExplainDataFrameAnalyticsAction;
import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction; import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction;
import org.elasticsearch.xpack.core.textstructure.action.FindFileStructureAction; import org.elasticsearch.xpack.core.textstructure.action.FindStructureAction;
import org.elasticsearch.xpack.core.ml.action.FlushJobAction; import org.elasticsearch.xpack.core.ml.action.FlushJobAction;
import org.elasticsearch.xpack.core.ml.action.ForecastJobAction; import org.elasticsearch.xpack.core.ml.action.ForecastJobAction;
import org.elasticsearch.xpack.core.ml.action.GetBucketsAction; import org.elasticsearch.xpack.core.ml.action.GetBucketsAction;
@ -409,7 +409,7 @@ public class XPackClientPlugin extends Plugin implements ActionPlugin, NetworkPl
GetAsyncSearchAction.INSTANCE, GetAsyncSearchAction.INSTANCE,
DeleteAsyncResultAction.INSTANCE, DeleteAsyncResultAction.INSTANCE,
// Text Structure // Text Structure
FindFileStructureAction.INSTANCE FindStructureAction.INSTANCE
)); ));
// rollupV2 // rollupV2

View file

@ -18,7 +18,7 @@ import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.StatusToXContentObject; import org.elasticsearch.common.xcontent.StatusToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FileStructure; import org.elasticsearch.xpack.core.textstructure.structurefinder.TextStructure;
import java.io.IOException; import java.io.IOException;
import java.util.Arrays; import java.util.Arrays;
@ -28,31 +28,31 @@ import java.util.Objects;
import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.action.ValidateActions.addValidationError;
public class FindFileStructureAction extends ActionType<FindFileStructureAction.Response> { public class FindStructureAction extends ActionType<FindStructureAction.Response> {
public static final FindFileStructureAction INSTANCE = new FindFileStructureAction(); public static final FindStructureAction INSTANCE = new FindStructureAction();
public static final String NAME = "cluster:monitor/text_structure/findstructure"; public static final String NAME = "cluster:monitor/text_structure/findstructure";
private FindFileStructureAction() { private FindStructureAction() {
super(NAME, Response::new); super(NAME, Response::new);
} }
public static class Response extends ActionResponse implements StatusToXContentObject, Writeable { public static class Response extends ActionResponse implements StatusToXContentObject, Writeable {
private FileStructure fileStructure; private TextStructure textStructure;
public Response(FileStructure fileStructure) { public Response(TextStructure textStructure) {
this.fileStructure = fileStructure; this.textStructure = textStructure;
} }
Response(StreamInput in) throws IOException { Response(StreamInput in) throws IOException {
super(in); super(in);
fileStructure = new FileStructure(in); textStructure = new TextStructure(in);
} }
@Override @Override
public void writeTo(StreamOutput out) throws IOException { public void writeTo(StreamOutput out) throws IOException {
fileStructure.writeTo(out); textStructure.writeTo(out);
} }
@Override @Override
@ -62,13 +62,13 @@ public class FindFileStructureAction extends ActionType<FindFileStructureAction.
@Override @Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
fileStructure.toXContent(builder, params); textStructure.toXContent(builder, params);
return builder; return builder;
} }
@Override @Override
public int hashCode() { public int hashCode() {
return Objects.hash(fileStructure); return Objects.hash(textStructure);
} }
@Override @Override
@ -82,8 +82,8 @@ public class FindFileStructureAction extends ActionType<FindFileStructureAction.
return false; return false;
} }
FindFileStructureAction.Response that = (FindFileStructureAction.Response) other; FindStructureAction.Response that = (FindStructureAction.Response) other;
return Objects.equals(fileStructure, that.fileStructure); return Objects.equals(textStructure, that.textStructure);
} }
} }
@ -92,17 +92,17 @@ public class FindFileStructureAction extends ActionType<FindFileStructureAction.
public static final ParseField LINES_TO_SAMPLE = new ParseField("lines_to_sample"); public static final ParseField LINES_TO_SAMPLE = new ParseField("lines_to_sample");
public static final ParseField LINE_MERGE_SIZE_LIMIT = new ParseField("line_merge_size_limit"); public static final ParseField LINE_MERGE_SIZE_LIMIT = new ParseField("line_merge_size_limit");
public static final ParseField TIMEOUT = new ParseField("timeout"); public static final ParseField TIMEOUT = new ParseField("timeout");
public static final ParseField CHARSET = FileStructure.CHARSET; public static final ParseField CHARSET = TextStructure.CHARSET;
public static final ParseField FORMAT = FileStructure.FORMAT; public static final ParseField FORMAT = TextStructure.FORMAT;
public static final ParseField COLUMN_NAMES = FileStructure.COLUMN_NAMES; public static final ParseField COLUMN_NAMES = TextStructure.COLUMN_NAMES;
public static final ParseField HAS_HEADER_ROW = FileStructure.HAS_HEADER_ROW; public static final ParseField HAS_HEADER_ROW = TextStructure.HAS_HEADER_ROW;
public static final ParseField DELIMITER = FileStructure.DELIMITER; public static final ParseField DELIMITER = TextStructure.DELIMITER;
public static final ParseField QUOTE = FileStructure.QUOTE; public static final ParseField QUOTE = TextStructure.QUOTE;
public static final ParseField SHOULD_TRIM_FIELDS = FileStructure.SHOULD_TRIM_FIELDS; public static final ParseField SHOULD_TRIM_FIELDS = TextStructure.SHOULD_TRIM_FIELDS;
public static final ParseField GROK_PATTERN = FileStructure.GROK_PATTERN; public static final ParseField GROK_PATTERN = TextStructure.GROK_PATTERN;
// This one is plural in FileStructure, but singular in FileStructureOverrides // This one is plural in FileStructure, but singular in FileStructureOverrides
public static final ParseField TIMESTAMP_FORMAT = new ParseField("timestamp_format"); public static final ParseField TIMESTAMP_FORMAT = new ParseField("timestamp_format");
public static final ParseField TIMESTAMP_FIELD = FileStructure.TIMESTAMP_FIELD; public static final ParseField TIMESTAMP_FIELD = TextStructure.TIMESTAMP_FIELD;
private static final String ARG_INCOMPATIBLE_WITH_FORMAT_TEMPLATE = private static final String ARG_INCOMPATIBLE_WITH_FORMAT_TEMPLATE =
"[%s] may only be specified if [" + FORMAT.getPreferredName() + "] is [%s]"; "[%s] may only be specified if [" + FORMAT.getPreferredName() + "] is [%s]";
@ -111,7 +111,7 @@ public class FindFileStructureAction extends ActionType<FindFileStructureAction.
private Integer lineMergeSizeLimit; private Integer lineMergeSizeLimit;
private TimeValue timeout; private TimeValue timeout;
private String charset; private String charset;
private FileStructure.Format format; private TextStructure.Format format;
private List<String> columnNames; private List<String> columnNames;
private Boolean hasHeaderRow; private Boolean hasHeaderRow;
private Character delimiter; private Character delimiter;
@ -131,7 +131,7 @@ public class FindFileStructureAction extends ActionType<FindFileStructureAction.
lineMergeSizeLimit = in.readOptionalVInt(); lineMergeSizeLimit = in.readOptionalVInt();
timeout = in.readOptionalTimeValue(); timeout = in.readOptionalTimeValue();
charset = in.readOptionalString(); charset = in.readOptionalString();
format = in.readBoolean() ? in.readEnum(FileStructure.Format.class) : null; format = in.readBoolean() ? in.readEnum(TextStructure.Format.class) : null;
columnNames = in.readBoolean() ? in.readStringList() : null; columnNames = in.readBoolean() ? in.readStringList() : null;
hasHeaderRow = in.readOptionalBoolean(); hasHeaderRow = in.readOptionalBoolean();
delimiter = in.readBoolean() ? (char) in.readVInt() : null; delimiter = in.readBoolean() ? (char) in.readVInt() : null;
@ -176,16 +176,16 @@ public class FindFileStructureAction extends ActionType<FindFileStructureAction.
this.charset = (charset == null || charset.isEmpty()) ? null : charset; this.charset = (charset == null || charset.isEmpty()) ? null : charset;
} }
public FileStructure.Format getFormat() { public TextStructure.Format getFormat() {
return format; return format;
} }
public void setFormat(FileStructure.Format format) { public void setFormat(TextStructure.Format format) {
this.format = format; this.format = format;
} }
public void setFormat(String format) { public void setFormat(String format) {
this.format = (format == null || format.isEmpty()) ? null : FileStructure.Format.fromString(format); this.format = (format == null || format.isEmpty()) ? null : TextStructure.Format.fromString(format);
} }
public List<String> getColumnNames() { public List<String> getColumnNames() {
@ -284,7 +284,7 @@ public class FindFileStructureAction extends ActionType<FindFileStructureAction.
this.sample = sample; this.sample = sample;
} }
private static ActionRequestValidationException addIncompatibleArgError(ParseField arg, FileStructure.Format format, private static ActionRequestValidationException addIncompatibleArgError(ParseField arg, TextStructure.Format format,
ActionRequestValidationException validationException) { ActionRequestValidationException validationException) {
return addValidationError(String.format(Locale.ROOT, ARG_INCOMPATIBLE_WITH_FORMAT_TEMPLATE, arg.getPreferredName(), format), return addValidationError(String.format(Locale.ROOT, ARG_INCOMPATIBLE_WITH_FORMAT_TEMPLATE, arg.getPreferredName(), format),
validationException); validationException);
@ -301,27 +301,27 @@ public class FindFileStructureAction extends ActionType<FindFileStructureAction.
validationException = addValidationError("[" + LINE_MERGE_SIZE_LIMIT.getPreferredName() + "] must be positive if specified", validationException = addValidationError("[" + LINE_MERGE_SIZE_LIMIT.getPreferredName() + "] must be positive if specified",
validationException); validationException);
} }
if (format != FileStructure.Format.DELIMITED) { if (format != TextStructure.Format.DELIMITED) {
if (columnNames != null) { if (columnNames != null) {
validationException = addIncompatibleArgError(COLUMN_NAMES, FileStructure.Format.DELIMITED, validationException); validationException = addIncompatibleArgError(COLUMN_NAMES, TextStructure.Format.DELIMITED, validationException);
} }
if (hasHeaderRow != null) { if (hasHeaderRow != null) {
validationException = addIncompatibleArgError(HAS_HEADER_ROW, FileStructure.Format.DELIMITED, validationException); validationException = addIncompatibleArgError(HAS_HEADER_ROW, TextStructure.Format.DELIMITED, validationException);
} }
if (delimiter != null) { if (delimiter != null) {
validationException = addIncompatibleArgError(DELIMITER, FileStructure.Format.DELIMITED, validationException); validationException = addIncompatibleArgError(DELIMITER, TextStructure.Format.DELIMITED, validationException);
} }
if (quote != null) { if (quote != null) {
validationException = addIncompatibleArgError(QUOTE, FileStructure.Format.DELIMITED, validationException); validationException = addIncompatibleArgError(QUOTE, TextStructure.Format.DELIMITED, validationException);
} }
if (shouldTrimFields != null) { if (shouldTrimFields != null) {
validationException = addIncompatibleArgError(SHOULD_TRIM_FIELDS, FileStructure.Format.DELIMITED, validationException); validationException = addIncompatibleArgError(SHOULD_TRIM_FIELDS, TextStructure.Format.DELIMITED, validationException);
} }
} }
if (format != FileStructure.Format.SEMI_STRUCTURED_TEXT) { if (format != TextStructure.Format.SEMI_STRUCTURED_TEXT) {
if (grokPattern != null) { if (grokPattern != null) {
validationException = validationException =
addIncompatibleArgError(GROK_PATTERN, FileStructure.Format.SEMI_STRUCTURED_TEXT, validationException); addIncompatibleArgError(GROK_PATTERN, TextStructure.Format.SEMI_STRUCTURED_TEXT, validationException);
} }
} }
if (sample == null || sample.length() == 0) { if (sample == null || sample.length() == 0) {

View file

@ -27,7 +27,7 @@ import java.util.TreeMap;
/** /**
* Stores the determined file format. * Stores the determined file format.
*/ */
public class FileStructure implements ToXContentObject, Writeable { public class TextStructure implements ToXContentObject, Writeable {
public enum Format { public enum Format {
@ -163,7 +163,7 @@ public class FileStructure implements ToXContentObject, Writeable {
private final SortedMap<String, FieldStats> fieldStats; private final SortedMap<String, FieldStats> fieldStats;
private final List<String> explanation; private final List<String> explanation;
public FileStructure(int numLinesAnalyzed, int numMessagesAnalyzed, String sampleStart, String charset, Boolean hasByteOrderMarker, public TextStructure(int numLinesAnalyzed, int numMessagesAnalyzed, String sampleStart, String charset, Boolean hasByteOrderMarker,
Format format, String multilineStartPattern, String excludeLinesPattern, List<String> columnNames, Format format, String multilineStartPattern, String excludeLinesPattern, List<String> columnNames,
Boolean hasHeaderRow, Character delimiter, Character quote, Boolean shouldTrimFields, String grokPattern, Boolean hasHeaderRow, Character delimiter, Character quote, Boolean shouldTrimFields, String grokPattern,
String timestampField, List<String> jodaTimestampFormats, List<String> javaTimestampFormats, String timestampField, List<String> jodaTimestampFormats, List<String> javaTimestampFormats,
@ -194,7 +194,7 @@ public class FileStructure implements ToXContentObject, Writeable {
this.explanation = List.copyOf(explanation); this.explanation = List.copyOf(explanation);
} }
public FileStructure(StreamInput in) throws IOException { public TextStructure(StreamInput in) throws IOException {
numLinesAnalyzed = in.readVInt(); numLinesAnalyzed = in.readVInt();
numMessagesAnalyzed = in.readVInt(); numMessagesAnalyzed = in.readVInt();
sampleStart = in.readString(); sampleStart = in.readString();
@ -447,7 +447,7 @@ public class FileStructure implements ToXContentObject, Writeable {
return false; return false;
} }
FileStructure that = (FileStructure) other; TextStructure that = (TextStructure) other;
return this.numLinesAnalyzed == that.numLinesAnalyzed && return this.numLinesAnalyzed == that.numLinesAnalyzed &&
this.numMessagesAnalyzed == that.numMessagesAnalyzed && this.numMessagesAnalyzed == that.numMessagesAnalyzed &&
Objects.equals(this.sampleStart, that.sampleStart) && Objects.equals(this.sampleStart, that.sampleStart) &&
@ -615,7 +615,7 @@ public class FileStructure implements ToXContentObject, Writeable {
} }
@SuppressWarnings("fallthrough") @SuppressWarnings("fallthrough")
public FileStructure build() { public TextStructure build() {
if (numLinesAnalyzed <= 0) { if (numLinesAnalyzed <= 0) {
throw new IllegalArgumentException("Number of lines analyzed must be positive."); throw new IllegalArgumentException("Number of lines analyzed must be positive.");
@ -728,7 +728,7 @@ public class FileStructure implements ToXContentObject, Writeable {
throw new IllegalArgumentException("Explanation must be specified."); throw new IllegalArgumentException("Explanation must be specified.");
} }
return new FileStructure(numLinesAnalyzed, numMessagesAnalyzed, sampleStart, charset, hasByteOrderMarker, format, return new TextStructure(numLinesAnalyzed, numMessagesAnalyzed, sampleStart, charset, hasByteOrderMarker, format,
multilineStartPattern, excludeLinesPattern, columnNames, hasHeaderRow, delimiter, quote, shouldTrimFields, grokPattern, multilineStartPattern, excludeLinesPattern, columnNames, hasHeaderRow, delimiter, quote, shouldTrimFields, grokPattern,
timestampField, jodaTimestampFormats, javaTimestampFormats, needClientTimezone, mappings, ingestPipeline, fieldStats, timestampField, jodaTimestampFormats, javaTimestampFormats, needClientTimezone, mappings, ingestPipeline, fieldStats,
explanation); explanation);

View file

@ -73,7 +73,7 @@ import org.elasticsearch.xpack.core.ml.action.EstimateModelMemoryAction;
import org.elasticsearch.xpack.core.ml.action.EvaluateDataFrameAction; import org.elasticsearch.xpack.core.ml.action.EvaluateDataFrameAction;
import org.elasticsearch.xpack.core.ml.action.ExplainDataFrameAnalyticsAction; import org.elasticsearch.xpack.core.ml.action.ExplainDataFrameAnalyticsAction;
import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction; import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction;
import org.elasticsearch.xpack.core.textstructure.action.FindFileStructureAction; import org.elasticsearch.xpack.core.textstructure.action.FindStructureAction;
import org.elasticsearch.xpack.core.ml.action.FlushJobAction; import org.elasticsearch.xpack.core.ml.action.FlushJobAction;
import org.elasticsearch.xpack.core.ml.action.ForecastJobAction; import org.elasticsearch.xpack.core.ml.action.ForecastJobAction;
import org.elasticsearch.xpack.core.ml.action.GetBucketsAction; import org.elasticsearch.xpack.core.ml.action.GetBucketsAction;
@ -356,7 +356,7 @@ public class ReservedRolesStoreTests extends ESTestCase {
assertRoleHasManageMl(kibanaRole); assertRoleHasManageMl(kibanaRole);
// Text Structure // Text Structure
assertThat(kibanaRole.cluster().check(FindFileStructureAction.NAME, request, authentication), is(true)); assertThat(kibanaRole.cluster().check(FindStructureAction.NAME, request, authentication), is(true));
// Application Privileges // Application Privileges
DeletePrivilegesRequest deleteKibanaPrivileges = new DeletePrivilegesRequest("kibana-.kibana", new String[]{ "all", "read" }); DeletePrivilegesRequest deleteKibanaPrivileges = new DeletePrivilegesRequest("kibana-.kibana", new String[]{ "all", "read" });

View file

@ -1,24 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.core.textstructure.action;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.test.AbstractWireSerializingTestCase;
import org.elasticsearch.xpack.core.textstructure.action.FindFileStructureAction;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FileStructureTests;
public class FindFileStructureActionResponseTests extends AbstractWireSerializingTestCase<FindFileStructureAction.Response> {
@Override
protected FindFileStructureAction.Response createTestInstance() {
return new FindFileStructureAction.Response(FileStructureTests.createTestFileStructure());
}
@Override
protected Writeable.Reader<FindFileStructureAction.Response> instanceReader() {
return FindFileStructureAction.Response::new;
}
}

View file

@ -9,19 +9,19 @@ import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.test.AbstractWireSerializingTestCase;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FileStructure; import org.elasticsearch.xpack.core.textstructure.structurefinder.TextStructure;
import java.util.Arrays; import java.util.Arrays;
import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.startsWith; import static org.hamcrest.Matchers.startsWith;
public class FindFileStructureActionRequestTests extends AbstractWireSerializingTestCase<FindFileStructureAction.Request> { public class FindTextStructureActionRequestTests extends AbstractWireSerializingTestCase<FindStructureAction.Request> {
@Override @Override
protected FindFileStructureAction.Request createTestInstance() { protected FindStructureAction.Request createTestInstance() {
FindFileStructureAction.Request request = new FindFileStructureAction.Request(); FindStructureAction.Request request = new FindStructureAction.Request();
if (randomBoolean()) { if (randomBoolean()) {
request.setLinesToSample(randomIntBetween(10, 2000)); request.setLinesToSample(randomIntBetween(10, 2000));
@ -36,9 +36,9 @@ public class FindFileStructureActionRequestTests extends AbstractWireSerializing
} }
if (randomBoolean()) { if (randomBoolean()) {
FileStructure.Format format = randomFrom(FileStructure.Format.values()); TextStructure.Format format = randomFrom(TextStructure.Format.values());
request.setFormat(format); request.setFormat(format);
if (format == FileStructure.Format.DELIMITED) { if (format == TextStructure.Format.DELIMITED) {
if (randomBoolean()) { if (randomBoolean()) {
request.setColumnNames(generateRandomStringArray(10, 15, false, false)); request.setColumnNames(generateRandomStringArray(10, 15, false, false));
} }
@ -54,7 +54,7 @@ public class FindFileStructureActionRequestTests extends AbstractWireSerializing
if (randomBoolean()) { if (randomBoolean()) {
request.setShouldTrimFields(randomBoolean()); request.setShouldTrimFields(randomBoolean());
} }
} else if (format == FileStructure.Format.SEMI_STRUCTURED_TEXT) { } else if (format == TextStructure.Format.SEMI_STRUCTURED_TEXT) {
if (randomBoolean()) { if (randomBoolean()) {
request.setGrokPattern(randomAlphaOfLength(80)); request.setGrokPattern(randomAlphaOfLength(80));
} }
@ -74,13 +74,13 @@ public class FindFileStructureActionRequestTests extends AbstractWireSerializing
} }
@Override @Override
protected Writeable.Reader<FindFileStructureAction.Request> instanceReader() { protected Writeable.Reader<FindStructureAction.Request> instanceReader() {
return FindFileStructureAction.Request::new; return FindStructureAction.Request::new;
} }
public void testValidateLinesToSample() { public void testValidateLinesToSample() {
FindFileStructureAction.Request request = new FindFileStructureAction.Request(); FindStructureAction.Request request = new FindStructureAction.Request();
request.setLinesToSample(randomIntBetween(-1, 0)); request.setLinesToSample(randomIntBetween(-1, 0));
request.setSample(new BytesArray("foo\n")); request.setSample(new BytesArray("foo\n"));
@ -92,7 +92,7 @@ public class FindFileStructureActionRequestTests extends AbstractWireSerializing
public void testValidateLineMergeSizeLimit() { public void testValidateLineMergeSizeLimit() {
FindFileStructureAction.Request request = new FindFileStructureAction.Request(); FindStructureAction.Request request = new FindStructureAction.Request();
request.setLineMergeSizeLimit(randomIntBetween(-1, 0)); request.setLineMergeSizeLimit(randomIntBetween(-1, 0));
request.setSample(new BytesArray("foo\n")); request.setSample(new BytesArray("foo\n"));
@ -104,7 +104,7 @@ public class FindFileStructureActionRequestTests extends AbstractWireSerializing
public void testValidateNonDelimited() { public void testValidateNonDelimited() {
FindFileStructureAction.Request request = new FindFileStructureAction.Request(); FindStructureAction.Request request = new FindStructureAction.Request();
String errorField; String errorField;
switch (randomIntBetween(0, 4)) { switch (randomIntBetween(0, 4)) {
case 0: case 0:
@ -140,8 +140,8 @@ public class FindFileStructureActionRequestTests extends AbstractWireSerializing
public void testValidateNonSemiStructuredText() { public void testValidateNonSemiStructuredText() {
FindFileStructureAction.Request request = new FindFileStructureAction.Request(); FindStructureAction.Request request = new FindStructureAction.Request();
request.setFormat(randomFrom(FileStructure.Format.NDJSON, FileStructure.Format.XML, FileStructure.Format.DELIMITED)); request.setFormat(randomFrom(TextStructure.Format.NDJSON, TextStructure.Format.XML, TextStructure.Format.DELIMITED));
request.setGrokPattern(randomAlphaOfLength(80)); request.setGrokPattern(randomAlphaOfLength(80));
request.setSample(new BytesArray("foo\n")); request.setSample(new BytesArray("foo\n"));
@ -153,7 +153,7 @@ public class FindFileStructureActionRequestTests extends AbstractWireSerializing
public void testValidateSample() { public void testValidateSample() {
FindFileStructureAction.Request request = new FindFileStructureAction.Request(); FindStructureAction.Request request = new FindStructureAction.Request();
if (randomBoolean()) { if (randomBoolean()) {
request.setSample(BytesArray.EMPTY); request.setSample(BytesArray.EMPTY);
} }

View file

@ -0,0 +1,23 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.core.textstructure.action;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.test.AbstractWireSerializingTestCase;
import org.elasticsearch.xpack.core.textstructure.structurefinder.TextStructureTests;
public class FindTextStructureActionResponseTests extends AbstractWireSerializingTestCase<FindStructureAction.Response> {
@Override
protected FindStructureAction.Response createTestInstance() {
return new FindStructureAction.Response(TextStructureTests.createTestFileStructure());
}
@Override
protected Writeable.Reader<FindStructureAction.Response> instanceReader() {
return FindStructureAction.Response::new;
}
}

View file

@ -19,18 +19,18 @@ import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.TreeMap; import java.util.TreeMap;
public class FileStructureTests extends AbstractSerializingTestCase<FileStructure> { public class TextStructureTests extends AbstractSerializingTestCase<TextStructure> {
@Override @Override
protected FileStructure createTestInstance() { protected TextStructure createTestInstance() {
return createTestFileStructure(); return createTestFileStructure();
} }
public static FileStructure createTestFileStructure() { public static TextStructure createTestFileStructure() {
FileStructure.Format format = randomFrom(EnumSet.allOf(FileStructure.Format.class)); TextStructure.Format format = randomFrom(EnumSet.allOf(TextStructure.Format.class));
FileStructure.Builder builder = new FileStructure.Builder(format); TextStructure.Builder builder = new TextStructure.Builder(format);
int numLinesAnalyzed = randomIntBetween(2, 10000); int numLinesAnalyzed = randomIntBetween(2, 10000);
builder.setNumLinesAnalyzed(numLinesAnalyzed); builder.setNumLinesAnalyzed(numLinesAnalyzed);
@ -51,18 +51,18 @@ public class FileStructureTests extends AbstractSerializingTestCase<FileStructur
builder.setExcludeLinesPattern(randomAlphaOfLength(100)); builder.setExcludeLinesPattern(randomAlphaOfLength(100));
} }
if (format == FileStructure.Format.DELIMITED) { if (format == TextStructure.Format.DELIMITED) {
builder.setColumnNames(Arrays.asList(generateRandomStringArray(10, 10, false, false))); builder.setColumnNames(Arrays.asList(generateRandomStringArray(10, 10, false, false)));
builder.setHasHeaderRow(randomBoolean()); builder.setHasHeaderRow(randomBoolean());
builder.setDelimiter(randomFrom(',', '\t', ';', '|')); builder.setDelimiter(randomFrom(',', '\t', ';', '|'));
builder.setQuote(randomFrom('"', '\'')); builder.setQuote(randomFrom('"', '\''));
} }
if (format == FileStructure.Format.SEMI_STRUCTURED_TEXT) { if (format == TextStructure.Format.SEMI_STRUCTURED_TEXT) {
builder.setGrokPattern(randomAlphaOfLength(100)); builder.setGrokPattern(randomAlphaOfLength(100));
} }
if (format == FileStructure.Format.SEMI_STRUCTURED_TEXT || randomBoolean()) { if (format == TextStructure.Format.SEMI_STRUCTURED_TEXT || randomBoolean()) {
builder.setTimestampField(randomAlphaOfLength(10)); builder.setTimestampField(randomAlphaOfLength(10));
builder.setJodaTimestampFormats(Arrays.asList(generateRandomStringArray(3, 20, false, false))); builder.setJodaTimestampFormats(Arrays.asList(generateRandomStringArray(3, 20, false, false)));
builder.setJavaTimestampFormats(Arrays.asList(generateRandomStringArray(3, 20, false, false))); builder.setJavaTimestampFormats(Arrays.asList(generateRandomStringArray(3, 20, false, false)));
@ -97,17 +97,17 @@ public class FileStructureTests extends AbstractSerializingTestCase<FileStructur
} }
@Override @Override
protected Writeable.Reader<FileStructure> instanceReader() { protected Writeable.Reader<TextStructure> instanceReader() {
return FileStructure::new; return TextStructure::new;
} }
@Override @Override
protected FileStructure doParseInstance(XContentParser parser) { protected TextStructure doParseInstance(XContentParser parser) {
return FileStructure.PARSER.apply(parser, null).build(); return TextStructure.PARSER.apply(parser, null).build();
} }
@Override @Override
protected ToXContent.Params getToXContentParams() { protected ToXContent.Params getToXContentParams() {
return new ToXContent.MapParams(Collections.singletonMap(FileStructure.EXPLAIN, "true")); return new ToXContent.MapParams(Collections.singletonMap(TextStructure.EXPLAIN, "true"));
} }
} }

View file

@ -1,7 +1,7 @@
{ {
"text_structure.find_structure":{ "text_structure.find_structure":{
"documentation":{ "documentation":{
"url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-find-file-structure.html", "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/find-structure.html",
"description":"Finds the structure of a text file. The text file must contain data that is suitable to be ingested into Elasticsearch." "description":"Finds the structure of a text file. The text file must contain data that is suitable to be ingested into Elasticsearch."
}, },
"stability":"experimental", "stability":"experimental",

View file

@ -18,9 +18,9 @@ import org.elasticsearch.plugins.ActionPlugin;
import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestController;
import org.elasticsearch.rest.RestHandler; import org.elasticsearch.rest.RestHandler;
import org.elasticsearch.xpack.core.textstructure.action.FindFileStructureAction; import org.elasticsearch.xpack.core.textstructure.action.FindStructureAction;
import org.elasticsearch.xpack.textstructure.rest.RestFindFileStructureAction; import org.elasticsearch.xpack.textstructure.rest.RestFindStructureAction;
import org.elasticsearch.xpack.textstructure.transport.TransportFindFileStructureAction; import org.elasticsearch.xpack.textstructure.transport.TransportFindStructureAction;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
@ -44,12 +44,12 @@ public class TextStructurePlugin extends Plugin implements ActionPlugin {
IndexNameExpressionResolver indexNameExpressionResolver, IndexNameExpressionResolver indexNameExpressionResolver,
Supplier<DiscoveryNodes> nodesInCluster Supplier<DiscoveryNodes> nodesInCluster
) { ) {
return Arrays.asList(new RestFindFileStructureAction()); return Arrays.asList(new RestFindStructureAction());
} }
@Override @Override
public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() { public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
return Arrays.asList(new ActionHandler<>(FindFileStructureAction.INSTANCE, TransportFindFileStructureAction.class)); return Arrays.asList(new ActionHandler<>(FindStructureAction.INSTANCE, TransportFindStructureAction.class));
} }
} }

View file

@ -11,8 +11,8 @@ import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.BaseRestHandler;
import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestRequest;
import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.rest.action.RestToXContentListener;
import org.elasticsearch.xpack.core.textstructure.action.FindFileStructureAction; import org.elasticsearch.xpack.core.textstructure.action.FindStructureAction;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FileStructure; import org.elasticsearch.xpack.core.textstructure.structurefinder.TextStructure;
import org.elasticsearch.xpack.textstructure.structurefinder.FileStructureFinderManager; import org.elasticsearch.xpack.textstructure.structurefinder.FileStructureFinderManager;
import java.util.Collections; import java.util.Collections;
@ -23,7 +23,7 @@ import java.util.concurrent.TimeUnit;
import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.elasticsearch.rest.RestRequest.Method.POST;
import static org.elasticsearch.xpack.textstructure.TextStructurePlugin.BASE_PATH; import static org.elasticsearch.xpack.textstructure.TextStructurePlugin.BASE_PATH;
public class RestFindFileStructureAction extends BaseRestHandler { public class RestFindStructureAction extends BaseRestHandler {
private static final TimeValue DEFAULT_TIMEOUT = new TimeValue(25, TimeUnit.SECONDS); private static final TimeValue DEFAULT_TIMEOUT = new TimeValue(25, TimeUnit.SECONDS);
@ -45,49 +45,47 @@ public class RestFindFileStructureAction extends BaseRestHandler {
@Override @Override
protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
FindFileStructureAction.Request request = new FindFileStructureAction.Request(); FindStructureAction.Request request = new FindStructureAction.Request();
request.setLinesToSample( request.setLinesToSample(
restRequest.paramAsInt( restRequest.paramAsInt(
FindFileStructureAction.Request.LINES_TO_SAMPLE.getPreferredName(), FindStructureAction.Request.LINES_TO_SAMPLE.getPreferredName(),
FileStructureFinderManager.DEFAULT_IDEAL_SAMPLE_LINE_COUNT FileStructureFinderManager.DEFAULT_IDEAL_SAMPLE_LINE_COUNT
) )
); );
request.setLineMergeSizeLimit( request.setLineMergeSizeLimit(
restRequest.paramAsInt( restRequest.paramAsInt(
FindFileStructureAction.Request.LINE_MERGE_SIZE_LIMIT.getPreferredName(), FindStructureAction.Request.LINE_MERGE_SIZE_LIMIT.getPreferredName(),
FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT
) )
); );
request.setTimeout( request.setTimeout(
TimeValue.parseTimeValue( TimeValue.parseTimeValue(
restRequest.param(FindFileStructureAction.Request.TIMEOUT.getPreferredName()), restRequest.param(FindStructureAction.Request.TIMEOUT.getPreferredName()),
DEFAULT_TIMEOUT, DEFAULT_TIMEOUT,
FindFileStructureAction.Request.TIMEOUT.getPreferredName() FindStructureAction.Request.TIMEOUT.getPreferredName()
) )
); );
request.setCharset(restRequest.param(FindFileStructureAction.Request.CHARSET.getPreferredName())); request.setCharset(restRequest.param(FindStructureAction.Request.CHARSET.getPreferredName()));
request.setFormat(restRequest.param(FindFileStructureAction.Request.FORMAT.getPreferredName())); request.setFormat(restRequest.param(FindStructureAction.Request.FORMAT.getPreferredName()));
request.setColumnNames(restRequest.paramAsStringArray(FindFileStructureAction.Request.COLUMN_NAMES.getPreferredName(), null)); request.setColumnNames(restRequest.paramAsStringArray(FindStructureAction.Request.COLUMN_NAMES.getPreferredName(), null));
request.setHasHeaderRow(restRequest.paramAsBoolean(FindFileStructureAction.Request.HAS_HEADER_ROW.getPreferredName(), null)); request.setHasHeaderRow(restRequest.paramAsBoolean(FindStructureAction.Request.HAS_HEADER_ROW.getPreferredName(), null));
request.setDelimiter(restRequest.param(FindFileStructureAction.Request.DELIMITER.getPreferredName())); request.setDelimiter(restRequest.param(FindStructureAction.Request.DELIMITER.getPreferredName()));
request.setQuote(restRequest.param(FindFileStructureAction.Request.QUOTE.getPreferredName())); request.setQuote(restRequest.param(FindStructureAction.Request.QUOTE.getPreferredName()));
request.setShouldTrimFields( request.setShouldTrimFields(restRequest.paramAsBoolean(FindStructureAction.Request.SHOULD_TRIM_FIELDS.getPreferredName(), null));
restRequest.paramAsBoolean(FindFileStructureAction.Request.SHOULD_TRIM_FIELDS.getPreferredName(), null) request.setGrokPattern(restRequest.param(FindStructureAction.Request.GROK_PATTERN.getPreferredName()));
); request.setTimestampFormat(restRequest.param(FindStructureAction.Request.TIMESTAMP_FORMAT.getPreferredName()));
request.setGrokPattern(restRequest.param(FindFileStructureAction.Request.GROK_PATTERN.getPreferredName())); request.setTimestampField(restRequest.param(FindStructureAction.Request.TIMESTAMP_FIELD.getPreferredName()));
request.setTimestampFormat(restRequest.param(FindFileStructureAction.Request.TIMESTAMP_FORMAT.getPreferredName()));
request.setTimestampField(restRequest.param(FindFileStructureAction.Request.TIMESTAMP_FIELD.getPreferredName()));
if (restRequest.hasContent()) { if (restRequest.hasContent()) {
request.setSample(restRequest.content()); request.setSample(restRequest.content());
} else { } else {
throw new ElasticsearchParseException("request body is required"); throw new ElasticsearchParseException("request body is required");
} }
return channel -> client.execute(FindFileStructureAction.INSTANCE, request, new RestToXContentListener<>(channel)); return channel -> client.execute(FindStructureAction.INSTANCE, request, new RestToXContentListener<>(channel));
} }
@Override @Override
protected Set<String> responseParams() { protected Set<String> responseParams() {
return Collections.singleton(FileStructure.EXPLAIN); return Collections.singleton(TextStructure.EXPLAIN);
} }
} }

View file

@ -8,7 +8,7 @@ package org.elasticsearch.xpack.textstructure.structurefinder;
import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.logging.log4j.message.ParameterizedMessage;
import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FieldStats; import org.elasticsearch.xpack.core.textstructure.structurefinder.FieldStats;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FileStructure; import org.elasticsearch.xpack.core.textstructure.structurefinder.TextStructure;
import org.supercsv.exception.SuperCsvException; import org.supercsv.exception.SuperCsvException;
import org.supercsv.io.CsvListReader; import org.supercsv.io.CsvListReader;
import org.supercsv.prefs.CsvPreference; import org.supercsv.prefs.CsvPreference;
@ -36,7 +36,7 @@ public class DelimitedFileStructureFinder implements FileStructureFinder {
private static final int MAX_LEVENSHTEIN_COMPARISONS = 100; private static final int MAX_LEVENSHTEIN_COMPARISONS = 100;
private static final int LONG_FIELD_THRESHOLD = 100; private static final int LONG_FIELD_THRESHOLD = 100;
private final List<String> sampleMessages; private final List<String> sampleMessages;
private final FileStructure structure; private final TextStructure structure;
static DelimitedFileStructureFinder makeDelimitedFileStructureFinder( static DelimitedFileStructureFinder makeDelimitedFileStructureFinder(
List<String> explanation, List<String> explanation,
@ -125,7 +125,7 @@ public class DelimitedFileStructureFinder implements FileStructureFinder {
Map<String, Object> csvProcessorSettings = makeCsvProcessorSettings("message", columnNamesList, delimiter, quoteChar, trimFields); Map<String, Object> csvProcessorSettings = makeCsvProcessorSettings("message", columnNamesList, delimiter, quoteChar, trimFields);
FileStructure.Builder structureBuilder = new FileStructure.Builder(FileStructure.Format.DELIMITED).setCharset(charsetName) TextStructure.Builder structureBuilder = new TextStructure.Builder(TextStructure.Format.DELIMITED).setCharset(charsetName)
.setHasByteOrderMarker(hasByteOrderMarker) .setHasByteOrderMarker(hasByteOrderMarker)
.setSampleStart(preamble) .setSampleStart(preamble)
.setNumLinesAnalyzed(lineNumbers.get(lineNumbers.size() - 1)) .setNumLinesAnalyzed(lineNumbers.get(lineNumbers.size() - 1))
@ -227,14 +227,14 @@ public class DelimitedFileStructureFinder implements FileStructureFinder {
structureBuilder.setFieldStats(mappingsAndFieldStats.v2()); structureBuilder.setFieldStats(mappingsAndFieldStats.v2());
} }
FileStructure structure = structureBuilder.setMappings( TextStructure structure = structureBuilder.setMappings(
Collections.singletonMap(FileStructureUtils.MAPPING_PROPERTIES_SETTING, fieldMappings) Collections.singletonMap(FileStructureUtils.MAPPING_PROPERTIES_SETTING, fieldMappings)
).setExplanation(explanation).build(); ).setExplanation(explanation).build();
return new DelimitedFileStructureFinder(sampleMessages, structure); return new DelimitedFileStructureFinder(sampleMessages, structure);
} }
private DelimitedFileStructureFinder(List<String> sampleMessages, FileStructure structure) { private DelimitedFileStructureFinder(List<String> sampleMessages, TextStructure structure) {
this.sampleMessages = Collections.unmodifiableList(sampleMessages); this.sampleMessages = Collections.unmodifiableList(sampleMessages);
this.structure = structure; this.structure = structure;
} }
@ -245,7 +245,7 @@ public class DelimitedFileStructureFinder implements FileStructureFinder {
} }
@Override @Override
public FileStructure getStructure() { public TextStructure getStructure() {
return structure; return structure;
} }

View file

@ -5,7 +5,7 @@
*/ */
package org.elasticsearch.xpack.textstructure.structurefinder; package org.elasticsearch.xpack.textstructure.structurefinder;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FileStructure; import org.elasticsearch.xpack.core.textstructure.structurefinder.TextStructure;
import org.supercsv.prefs.CsvPreference; import org.supercsv.prefs.CsvPreference;
import java.io.IOException; import java.io.IOException;
@ -37,8 +37,8 @@ public class DelimitedFileStructureFinderFactory implements FileStructureFinderF
} }
@Override @Override
public boolean canFindFormat(FileStructure.Format format) { public boolean canFindFormat(TextStructure.Format format) {
return format == null || format == FileStructure.Format.DELIMITED; return format == null || format == TextStructure.Format.DELIMITED;
} }
/** /**

View file

@ -5,7 +5,7 @@
*/ */
package org.elasticsearch.xpack.textstructure.structurefinder; package org.elasticsearch.xpack.textstructure.structurefinder;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FileStructure; import org.elasticsearch.xpack.core.textstructure.structurefinder.TextStructure;
import java.util.List; import java.util.List;
@ -21,5 +21,5 @@ public interface FileStructureFinder {
* Retrieve the structure of the file used to instantiate the finder. * Retrieve the structure of the file used to instantiate the finder.
* @return The file structure. * @return The file structure.
*/ */
FileStructure getStructure(); TextStructure getStructure();
} }

View file

@ -5,7 +5,7 @@
*/ */
package org.elasticsearch.xpack.textstructure.structurefinder; package org.elasticsearch.xpack.textstructure.structurefinder;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FileStructure; import org.elasticsearch.xpack.core.textstructure.structurefinder.TextStructure;
import java.util.List; import java.util.List;
@ -17,7 +17,7 @@ public interface FileStructureFinderFactory {
* @return <code>true</code> if {@code format} is <code>null</code> or the factory * @return <code>true</code> if {@code format} is <code>null</code> or the factory
* can produce a {@link FileStructureFinder} that can find {@code format}. * can produce a {@link FileStructureFinder} that can find {@code format}.
*/ */
boolean canFindFormat(FileStructure.Format format); boolean canFindFormat(TextStructure.Format format);
/** /**
* Given a sample of a file, decide whether this factory will be able * Given a sample of a file, decide whether this factory will be able

View file

@ -11,7 +11,7 @@ import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FileStructure; import org.elasticsearch.xpack.core.textstructure.structurefinder.TextStructure;
import java.io.BufferedInputStream; import java.io.BufferedInputStream;
import java.io.BufferedReader; import java.io.BufferedReader;
@ -573,7 +573,7 @@ public final class FileStructureFinderManager {
) )
); );
} else if (quote != null || shouldTrimFields != null || FileStructure.Format.DELIMITED.equals(overrides.getFormat())) { } else if (quote != null || shouldTrimFields != null || TextStructure.Format.DELIMITED.equals(overrides.getFormat())) {
allowedFractionOfBadLines = DelimitedFileStructureFinderFactory.FORMAT_OVERRIDDEN_ALLOWED_FRACTION_OF_BAD_LINES; allowedFractionOfBadLines = DelimitedFileStructureFinderFactory.FORMAT_OVERRIDDEN_ALLOWED_FRACTION_OF_BAD_LINES;
// The delimiter is not specified, but some other aspect of delimited files is, // The delimiter is not specified, but some other aspect of delimited files is,

View file

@ -5,8 +5,8 @@
*/ */
package org.elasticsearch.xpack.textstructure.structurefinder; package org.elasticsearch.xpack.textstructure.structurefinder;
import org.elasticsearch.xpack.core.textstructure.action.FindFileStructureAction; import org.elasticsearch.xpack.core.textstructure.action.FindStructureAction;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FileStructure; import org.elasticsearch.xpack.core.textstructure.structurefinder.TextStructure;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
@ -17,14 +17,14 @@ import java.util.Objects;
* aspect of the file structure detection is not overridden. * aspect of the file structure detection is not overridden.
* *
* There is no consistency checking in this class. Consistency checking of the different * There is no consistency checking in this class. Consistency checking of the different
* fields is done in {@link FindFileStructureAction.Request}. * fields is done in {@link FindStructureAction.Request}.
*/ */
public class FileStructureOverrides { public class FileStructureOverrides {
public static final FileStructureOverrides EMPTY_OVERRIDES = new Builder().build(); public static final FileStructureOverrides EMPTY_OVERRIDES = new Builder().build();
private final String charset; private final String charset;
private final FileStructure.Format format; private final TextStructure.Format format;
private final List<String> columnNames; private final List<String> columnNames;
private final Boolean hasHeaderRow; private final Boolean hasHeaderRow;
private final Character delimiter; private final Character delimiter;
@ -34,7 +34,7 @@ public class FileStructureOverrides {
private final String timestampFormat; private final String timestampFormat;
private final String timestampField; private final String timestampField;
public FileStructureOverrides(FindFileStructureAction.Request request) { public FileStructureOverrides(FindStructureAction.Request request) {
this( this(
request.getCharset(), request.getCharset(),
@ -52,7 +52,7 @@ public class FileStructureOverrides {
private FileStructureOverrides( private FileStructureOverrides(
String charset, String charset,
FileStructure.Format format, TextStructure.Format format,
List<String> columnNames, List<String> columnNames,
Boolean hasHeaderRow, Boolean hasHeaderRow,
Character delimiter, Character delimiter,
@ -82,7 +82,7 @@ public class FileStructureOverrides {
return charset; return charset;
} }
public FileStructure.Format getFormat() { public TextStructure.Format getFormat() {
return format; return format;
} }
@ -162,7 +162,7 @@ public class FileStructureOverrides {
public static class Builder { public static class Builder {
private String charset; private String charset;
private FileStructure.Format format; private TextStructure.Format format;
private List<String> columnNames; private List<String> columnNames;
private Boolean hasHeaderRow; private Boolean hasHeaderRow;
private Character delimiter; private Character delimiter;
@ -177,7 +177,7 @@ public class FileStructureOverrides {
return this; return this;
} }
public Builder setFormat(FileStructure.Format format) { public Builder setFormat(TextStructure.Format format) {
this.format = format; this.format = format;
return this; return this;
} }

View file

@ -10,7 +10,7 @@ import org.elasticsearch.common.xcontent.DeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FieldStats; import org.elasticsearch.xpack.core.textstructure.structurefinder.FieldStats;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FileStructure; import org.elasticsearch.xpack.core.textstructure.structurefinder.TextStructure;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
@ -29,7 +29,7 @@ import static org.elasticsearch.common.xcontent.json.JsonXContent.jsonXContent;
public class NdJsonFileStructureFinder implements FileStructureFinder { public class NdJsonFileStructureFinder implements FileStructureFinder {
private final List<String> sampleMessages; private final List<String> sampleMessages;
private final FileStructure structure; private final TextStructure structure;
static NdJsonFileStructureFinder makeNdJsonFileStructureFinder( static NdJsonFileStructureFinder makeNdJsonFileStructureFinder(
List<String> explanation, List<String> explanation,
@ -53,7 +53,7 @@ public class NdJsonFileStructureFinder implements FileStructureFinder {
timeoutChecker.check("NDJSON parsing"); timeoutChecker.check("NDJSON parsing");
} }
FileStructure.Builder structureBuilder = new FileStructure.Builder(FileStructure.Format.NDJSON).setCharset(charsetName) TextStructure.Builder structureBuilder = new TextStructure.Builder(TextStructure.Format.NDJSON).setCharset(charsetName)
.setHasByteOrderMarker(hasByteOrderMarker) .setHasByteOrderMarker(hasByteOrderMarker)
.setSampleStart(sampleMessages.stream().limit(2).collect(Collectors.joining("\n", "", "\n"))) .setSampleStart(sampleMessages.stream().limit(2).collect(Collectors.joining("\n", "", "\n")))
.setNumLinesAnalyzed(sampleMessages.size()) .setNumLinesAnalyzed(sampleMessages.size())
@ -100,14 +100,14 @@ public class NdJsonFileStructureFinder implements FileStructureFinder {
structureBuilder.setFieldStats(mappingsAndFieldStats.v2()); structureBuilder.setFieldStats(mappingsAndFieldStats.v2());
} }
FileStructure structure = structureBuilder.setMappings( TextStructure structure = structureBuilder.setMappings(
Collections.singletonMap(FileStructureUtils.MAPPING_PROPERTIES_SETTING, fieldMappings) Collections.singletonMap(FileStructureUtils.MAPPING_PROPERTIES_SETTING, fieldMappings)
).setExplanation(explanation).build(); ).setExplanation(explanation).build();
return new NdJsonFileStructureFinder(sampleMessages, structure); return new NdJsonFileStructureFinder(sampleMessages, structure);
} }
private NdJsonFileStructureFinder(List<String> sampleMessages, FileStructure structure) { private NdJsonFileStructureFinder(List<String> sampleMessages, TextStructure structure) {
this.sampleMessages = Collections.unmodifiableList(sampleMessages); this.sampleMessages = Collections.unmodifiableList(sampleMessages);
this.structure = structure; this.structure = structure;
} }
@ -118,7 +118,7 @@ public class NdJsonFileStructureFinder implements FileStructureFinder {
} }
@Override @Override
public FileStructure getStructure() { public TextStructure getStructure() {
return structure; return structure;
} }
} }

View file

@ -8,7 +8,7 @@ package org.elasticsearch.xpack.textstructure.structurefinder;
import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.DeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FileStructure; import org.elasticsearch.xpack.core.textstructure.structurefinder.TextStructure;
import java.io.IOException; import java.io.IOException;
import java.io.StringReader; import java.io.StringReader;
@ -20,8 +20,8 @@ import static org.elasticsearch.common.xcontent.json.JsonXContent.jsonXContent;
public class NdJsonFileStructureFinderFactory implements FileStructureFinderFactory { public class NdJsonFileStructureFinderFactory implements FileStructureFinderFactory {
@Override @Override
public boolean canFindFormat(FileStructure.Format format) { public boolean canFindFormat(TextStructure.Format format) {
return format == null || format == FileStructure.Format.NDJSON; return format == null || format == TextStructure.Format.NDJSON;
} }
/** /**

View file

@ -6,9 +6,9 @@
package org.elasticsearch.xpack.textstructure.structurefinder; package org.elasticsearch.xpack.textstructure.structurefinder;
import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.xpack.core.textstructure.action.FindFileStructureAction; import org.elasticsearch.xpack.core.textstructure.action.FindStructureAction;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FieldStats; import org.elasticsearch.xpack.core.textstructure.structurefinder.FieldStats;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FileStructure; import org.elasticsearch.xpack.core.textstructure.structurefinder.TextStructure;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
@ -22,7 +22,7 @@ import java.util.regex.Pattern;
public class TextLogFileStructureFinder implements FileStructureFinder { public class TextLogFileStructureFinder implements FileStructureFinder {
private final List<String> sampleMessages; private final List<String> sampleMessages;
private final FileStructure structure; private final TextStructure structure;
static TextLogFileStructureFinder makeTextLogFileStructureFinder( static TextLogFileStructureFinder makeTextLogFileStructureFinder(
List<String> explanation, List<String> explanation,
@ -98,7 +98,7 @@ public class TextLogFileStructureFinder implements FileStructureFinder {
+ lineMergeSizeLimit + lineMergeSizeLimit
+ "]). If you have messages this big please increase " + "]). If you have messages this big please increase "
+ "the value of [" + "the value of ["
+ FindFileStructureAction.Request.LINE_MERGE_SIZE_LIMIT + FindStructureAction.Request.LINE_MERGE_SIZE_LIMIT
+ "]. Otherwise it " + "]. Otherwise it "
+ "probably means the timestamp has been incorrectly detected, so try overriding that." + "probably means the timestamp has been incorrectly detected, so try overriding that."
); );
@ -125,7 +125,7 @@ public class TextLogFileStructureFinder implements FileStructureFinder {
// null to allow GC before Grok pattern search // null to allow GC before Grok pattern search
sampleLines = null; sampleLines = null;
FileStructure.Builder structureBuilder = new FileStructure.Builder(FileStructure.Format.SEMI_STRUCTURED_TEXT).setCharset( TextStructure.Builder structureBuilder = new TextStructure.Builder(TextStructure.Format.SEMI_STRUCTURED_TEXT).setCharset(
charsetName charsetName
) )
.setHasByteOrderMarker(hasByteOrderMarker) .setHasByteOrderMarker(hasByteOrderMarker)
@ -178,7 +178,7 @@ public class TextLogFileStructureFinder implements FileStructureFinder {
boolean needClientTimeZone = timestampFormatFinder.hasTimezoneDependentParsing(); boolean needClientTimeZone = timestampFormatFinder.hasTimezoneDependentParsing();
FileStructure structure = structureBuilder.setTimestampField(interimTimestampField) TextStructure structure = structureBuilder.setTimestampField(interimTimestampField)
.setJodaTimestampFormats(timestampFormatFinder.getJodaTimestampFormats()) .setJodaTimestampFormats(timestampFormatFinder.getJodaTimestampFormats())
.setJavaTimestampFormats(timestampFormatFinder.getJavaTimestampFormats()) .setJavaTimestampFormats(timestampFormatFinder.getJavaTimestampFormats())
.setNeedClientTimezone(needClientTimeZone) .setNeedClientTimezone(needClientTimeZone)
@ -203,7 +203,7 @@ public class TextLogFileStructureFinder implements FileStructureFinder {
return new TextLogFileStructureFinder(sampleMessages, structure); return new TextLogFileStructureFinder(sampleMessages, structure);
} }
private TextLogFileStructureFinder(List<String> sampleMessages, FileStructure structure) { private TextLogFileStructureFinder(List<String> sampleMessages, TextStructure structure) {
this.sampleMessages = Collections.unmodifiableList(sampleMessages); this.sampleMessages = Collections.unmodifiableList(sampleMessages);
this.structure = structure; this.structure = structure;
} }
@ -214,7 +214,7 @@ public class TextLogFileStructureFinder implements FileStructureFinder {
} }
@Override @Override
public FileStructure getStructure() { public TextStructure getStructure() {
return structure; return structure;
} }

View file

@ -5,7 +5,7 @@
*/ */
package org.elasticsearch.xpack.textstructure.structurefinder; package org.elasticsearch.xpack.textstructure.structurefinder;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FileStructure; import org.elasticsearch.xpack.core.textstructure.structurefinder.TextStructure;
import java.util.List; import java.util.List;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@ -16,8 +16,8 @@ public class TextLogFileStructureFinderFactory implements FileStructureFinderFac
private static final Pattern TWO_NON_BLANK_LINES_PATTERN = Pattern.compile(".\n+."); private static final Pattern TWO_NON_BLANK_LINES_PATTERN = Pattern.compile(".\n+.");
@Override @Override
public boolean canFindFormat(FileStructure.Format format) { public boolean canFindFormat(TextStructure.Format format) {
return format == null || format == FileStructure.Format.SEMI_STRUCTURED_TEXT; return format == null || format == TextStructure.Format.SEMI_STRUCTURED_TEXT;
} }
/** /**

View file

@ -7,7 +7,7 @@ package org.elasticsearch.xpack.textstructure.structurefinder;
import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FieldStats; import org.elasticsearch.xpack.core.textstructure.structurefinder.FieldStats;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FileStructure; import org.elasticsearch.xpack.core.textstructure.structurefinder.TextStructure;
import org.w3c.dom.Document; import org.w3c.dom.Document;
import org.w3c.dom.NamedNodeMap; import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node; import org.w3c.dom.Node;
@ -35,7 +35,7 @@ import java.util.regex.Pattern;
public class XmlFileStructureFinder implements FileStructureFinder { public class XmlFileStructureFinder implements FileStructureFinder {
private final List<String> sampleMessages; private final List<String> sampleMessages;
private final FileStructure structure; private final TextStructure structure;
static XmlFileStructureFinder makeXmlFileStructureFinder( static XmlFileStructureFinder makeXmlFileStructureFinder(
List<String> explanation, List<String> explanation,
@ -89,7 +89,7 @@ public class XmlFileStructureFinder implements FileStructureFinder {
assert messagePrefix.charAt(0) == '<'; assert messagePrefix.charAt(0) == '<';
String topLevelTag = messagePrefix.substring(1); String topLevelTag = messagePrefix.substring(1);
FileStructure.Builder structureBuilder = new FileStructure.Builder(FileStructure.Format.XML).setCharset(charsetName) TextStructure.Builder structureBuilder = new TextStructure.Builder(TextStructure.Format.XML).setCharset(charsetName)
.setHasByteOrderMarker(hasByteOrderMarker) .setHasByteOrderMarker(hasByteOrderMarker)
.setSampleStart(preamble.toString()) .setSampleStart(preamble.toString())
.setNumLinesAnalyzed(linesConsumed) .setNumLinesAnalyzed(linesConsumed)
@ -140,7 +140,7 @@ public class XmlFileStructureFinder implements FileStructureFinder {
outerFieldMappings.put(FileStructureUtils.DEFAULT_TIMESTAMP_FIELD, timeField.v2().getEsDateMappingTypeWithoutFormat()); outerFieldMappings.put(FileStructureUtils.DEFAULT_TIMESTAMP_FIELD, timeField.v2().getEsDateMappingTypeWithoutFormat());
} }
FileStructure structure = structureBuilder.setMappings( TextStructure structure = structureBuilder.setMappings(
Collections.singletonMap(FileStructureUtils.MAPPING_PROPERTIES_SETTING, outerFieldMappings) Collections.singletonMap(FileStructureUtils.MAPPING_PROPERTIES_SETTING, outerFieldMappings)
).setExplanation(explanation).build(); ).setExplanation(explanation).build();
@ -166,7 +166,7 @@ public class XmlFileStructureFinder implements FileStructureFinder {
return docBuilderFactory; return docBuilderFactory;
} }
private XmlFileStructureFinder(List<String> sampleMessages, FileStructure structure) { private XmlFileStructureFinder(List<String> sampleMessages, TextStructure structure) {
this.sampleMessages = Collections.unmodifiableList(sampleMessages); this.sampleMessages = Collections.unmodifiableList(sampleMessages);
this.structure = structure; this.structure = structure;
} }
@ -177,7 +177,7 @@ public class XmlFileStructureFinder implements FileStructureFinder {
} }
@Override @Override
public FileStructure getStructure() { public TextStructure getStructure() {
return structure; return structure;
} }

View file

@ -5,7 +5,7 @@
*/ */
package org.elasticsearch.xpack.textstructure.structurefinder; package org.elasticsearch.xpack.textstructure.structurefinder;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FileStructure; import org.elasticsearch.xpack.core.textstructure.structurefinder.TextStructure;
import org.xml.sax.SAXException; import org.xml.sax.SAXException;
import javax.xml.parsers.ParserConfigurationException; import javax.xml.parsers.ParserConfigurationException;
@ -32,8 +32,8 @@ public class XmlFileStructureFinderFactory implements FileStructureFinderFactory
} }
@Override @Override
public boolean canFindFormat(FileStructure.Format format) { public boolean canFindFormat(TextStructure.Format format) {
return format == null || format == FileStructure.Format.XML; return format == null || format == TextStructure.Format.XML;
} }
/** /**

View file

@ -12,7 +12,7 @@ import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.Task;
import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService; import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.core.textstructure.action.FindFileStructureAction; import org.elasticsearch.xpack.core.textstructure.action.FindStructureAction;
import org.elasticsearch.xpack.textstructure.structurefinder.FileStructureFinder; import org.elasticsearch.xpack.textstructure.structurefinder.FileStructureFinder;
import org.elasticsearch.xpack.textstructure.structurefinder.FileStructureFinderManager; import org.elasticsearch.xpack.textstructure.structurefinder.FileStructureFinderManager;
import org.elasticsearch.xpack.textstructure.structurefinder.FileStructureOverrides; import org.elasticsearch.xpack.textstructure.structurefinder.FileStructureOverrides;
@ -21,24 +21,18 @@ import java.io.InputStream;
import static org.elasticsearch.threadpool.ThreadPool.Names.GENERIC; import static org.elasticsearch.threadpool.ThreadPool.Names.GENERIC;
public class TransportFindFileStructureAction extends HandledTransportAction< public class TransportFindStructureAction extends HandledTransportAction<FindStructureAction.Request, FindStructureAction.Response> {
FindFileStructureAction.Request,
FindFileStructureAction.Response> {
private final ThreadPool threadPool; private final ThreadPool threadPool;
@Inject @Inject
public TransportFindFileStructureAction(TransportService transportService, ActionFilters actionFilters, ThreadPool threadPool) { public TransportFindStructureAction(TransportService transportService, ActionFilters actionFilters, ThreadPool threadPool) {
super(FindFileStructureAction.NAME, transportService, actionFilters, FindFileStructureAction.Request::new); super(FindStructureAction.NAME, transportService, actionFilters, FindStructureAction.Request::new);
this.threadPool = threadPool; this.threadPool = threadPool;
} }
@Override @Override
protected void doExecute( protected void doExecute(Task task, FindStructureAction.Request request, ActionListener<FindStructureAction.Response> listener) {
Task task,
FindFileStructureAction.Request request,
ActionListener<FindFileStructureAction.Response> listener
) {
// As determining the file structure might take a while, we run // As determining the file structure might take a while, we run
// in a different thread to avoid blocking the network thread. // in a different thread to avoid blocking the network thread.
@ -51,7 +45,7 @@ public class TransportFindFileStructureAction extends HandledTransportAction<
}); });
} }
private FindFileStructureAction.Response buildFileStructureResponse(FindFileStructureAction.Request request) throws Exception { private FindStructureAction.Response buildFileStructureResponse(FindStructureAction.Request request) throws Exception {
FileStructureFinderManager structureFinderManager = new FileStructureFinderManager(threadPool.scheduler()); FileStructureFinderManager structureFinderManager = new FileStructureFinderManager(threadPool.scheduler());
@ -64,7 +58,7 @@ public class TransportFindFileStructureAction extends HandledTransportAction<
request.getTimeout() request.getTimeout()
); );
return new FindFileStructureAction.Response(fileStructureFinder.getStructure()); return new FindStructureAction.Response(fileStructureFinder.getStructure());
} }
} }
} }

View file

@ -5,7 +5,7 @@
*/ */
package org.elasticsearch.xpack.textstructure.structurefinder; package org.elasticsearch.xpack.textstructure.structurefinder;
public class DelimitedFileStructureFinderFactoryTests extends FileStructureTestCase { public class DelimitedTextStructureFinderFactoryTests extends TextStructureTestCase {
private FileStructureFinderFactory csvFactory = new DelimitedFileStructureFinderFactory(',', '"', 2, false); private FileStructureFinderFactory csvFactory = new DelimitedFileStructureFinderFactory(',', '"', 2, false);
private FileStructureFinderFactory tsvFactory = new DelimitedFileStructureFinderFactory('\t', '"', 2, false); private FileStructureFinderFactory tsvFactory = new DelimitedFileStructureFinderFactory('\t', '"', 2, false);

View file

@ -6,7 +6,7 @@
package org.elasticsearch.xpack.textstructure.structurefinder; package org.elasticsearch.xpack.textstructure.structurefinder;
import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FileStructure; import org.elasticsearch.xpack.core.textstructure.structurefinder.TextStructure;
import org.supercsv.prefs.CsvPreference; import org.supercsv.prefs.CsvPreference;
import java.io.IOException; import java.io.IOException;
@ -28,7 +28,7 @@ import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasKey;
import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.not;
public class DelimitedFileStructureFinderTests extends FileStructureTestCase { public class DelimitedTextStructureFinderTests extends TextStructureTestCase {
private final FileStructureFinderFactory csvFactory = new DelimitedFileStructureFinderFactory(',', '"', 2, false); private final FileStructureFinderFactory csvFactory = new DelimitedFileStructureFinderFactory(',', '"', 2, false);
private final FileStructureFinderFactory tsvFactory = new DelimitedFileStructureFinderFactory('\t', '"', 3, false); private final FileStructureFinderFactory tsvFactory = new DelimitedFileStructureFinderFactory('\t', '"', 3, false);
@ -49,9 +49,9 @@ public class DelimitedFileStructureFinderTests extends FileStructureTestCase {
NOOP_TIMEOUT_CHECKER NOOP_TIMEOUT_CHECKER
); );
FileStructure structure = structureFinder.getStructure(); TextStructure structure = structureFinder.getStructure();
assertEquals(FileStructure.Format.DELIMITED, structure.getFormat()); assertEquals(TextStructure.Format.DELIMITED, structure.getFormat());
assertEquals(charset, structure.getCharset()); assertEquals(charset, structure.getCharset());
if (hasByteOrderMarker == null) { if (hasByteOrderMarker == null) {
assertNull(structure.getHasByteOrderMarker()); assertNull(structure.getHasByteOrderMarker());
@ -99,9 +99,9 @@ public class DelimitedFileStructureFinderTests extends FileStructureTestCase {
NOOP_TIMEOUT_CHECKER NOOP_TIMEOUT_CHECKER
); );
FileStructure structure = structureFinder.getStructure(); TextStructure structure = structureFinder.getStructure();
assertEquals(FileStructure.Format.DELIMITED, structure.getFormat()); assertEquals(TextStructure.Format.DELIMITED, structure.getFormat());
assertEquals(charset, structure.getCharset()); assertEquals(charset, structure.getCharset());
if (hasByteOrderMarker == null) { if (hasByteOrderMarker == null) {
assertNull(structure.getHasByteOrderMarker()); assertNull(structure.getHasByteOrderMarker());
@ -150,9 +150,9 @@ public class DelimitedFileStructureFinderTests extends FileStructureTestCase {
NOOP_TIMEOUT_CHECKER NOOP_TIMEOUT_CHECKER
); );
FileStructure structure = structureFinder.getStructure(); TextStructure structure = structureFinder.getStructure();
assertEquals(FileStructure.Format.DELIMITED, structure.getFormat()); assertEquals(TextStructure.Format.DELIMITED, structure.getFormat());
assertEquals(charset, structure.getCharset()); assertEquals(charset, structure.getCharset());
if (hasByteOrderMarker == null) { if (hasByteOrderMarker == null) {
assertNull(structure.getHasByteOrderMarker()); assertNull(structure.getHasByteOrderMarker());
@ -192,9 +192,9 @@ public class DelimitedFileStructureFinderTests extends FileStructureTestCase {
NOOP_TIMEOUT_CHECKER NOOP_TIMEOUT_CHECKER
); );
FileStructure structure = structureFinder.getStructure(); TextStructure structure = structureFinder.getStructure();
assertEquals(FileStructure.Format.DELIMITED, structure.getFormat()); assertEquals(TextStructure.Format.DELIMITED, structure.getFormat());
assertEquals(charset, structure.getCharset()); assertEquals(charset, structure.getCharset());
if (hasByteOrderMarker == null) { if (hasByteOrderMarker == null) {
assertNull(structure.getHasByteOrderMarker()); assertNull(structure.getHasByteOrderMarker());
@ -235,9 +235,9 @@ public class DelimitedFileStructureFinderTests extends FileStructureTestCase {
NOOP_TIMEOUT_CHECKER NOOP_TIMEOUT_CHECKER
); );
FileStructure structure = structureFinder.getStructure(); TextStructure structure = structureFinder.getStructure();
assertEquals(FileStructure.Format.DELIMITED, structure.getFormat()); assertEquals(TextStructure.Format.DELIMITED, structure.getFormat());
assertEquals(charset, structure.getCharset()); assertEquals(charset, structure.getCharset());
if (hasByteOrderMarker == null) { if (hasByteOrderMarker == null) {
assertNull(structure.getHasByteOrderMarker()); assertNull(structure.getHasByteOrderMarker());
@ -274,9 +274,9 @@ public class DelimitedFileStructureFinderTests extends FileStructureTestCase {
NOOP_TIMEOUT_CHECKER NOOP_TIMEOUT_CHECKER
); );
FileStructure structure = structureFinder.getStructure(); TextStructure structure = structureFinder.getStructure();
assertEquals(FileStructure.Format.DELIMITED, structure.getFormat()); assertEquals(TextStructure.Format.DELIMITED, structure.getFormat());
assertEquals(charset, structure.getCharset()); assertEquals(charset, structure.getCharset());
if (hasByteOrderMarker == null) { if (hasByteOrderMarker == null) {
assertNull(structure.getHasByteOrderMarker()); assertNull(structure.getHasByteOrderMarker());
@ -317,9 +317,9 @@ public class DelimitedFileStructureFinderTests extends FileStructureTestCase {
NOOP_TIMEOUT_CHECKER NOOP_TIMEOUT_CHECKER
); );
FileStructure structure = structureFinder.getStructure(); TextStructure structure = structureFinder.getStructure();
assertEquals(FileStructure.Format.DELIMITED, structure.getFormat()); assertEquals(TextStructure.Format.DELIMITED, structure.getFormat());
assertEquals(charset, structure.getCharset()); assertEquals(charset, structure.getCharset());
if (hasByteOrderMarker == null) { if (hasByteOrderMarker == null) {
assertNull(structure.getHasByteOrderMarker()); assertNull(structure.getHasByteOrderMarker());
@ -394,9 +394,9 @@ public class DelimitedFileStructureFinderTests extends FileStructureTestCase {
NOOP_TIMEOUT_CHECKER NOOP_TIMEOUT_CHECKER
); );
FileStructure structure = structureFinder.getStructure(); TextStructure structure = structureFinder.getStructure();
assertEquals(FileStructure.Format.DELIMITED, structure.getFormat()); assertEquals(TextStructure.Format.DELIMITED, structure.getFormat());
assertEquals(charset, structure.getCharset()); assertEquals(charset, structure.getCharset());
if (hasByteOrderMarker == null) { if (hasByteOrderMarker == null) {
assertNull(structure.getHasByteOrderMarker()); assertNull(structure.getHasByteOrderMarker());
@ -466,9 +466,9 @@ public class DelimitedFileStructureFinderTests extends FileStructureTestCase {
NOOP_TIMEOUT_CHECKER NOOP_TIMEOUT_CHECKER
); );
FileStructure structure = structureFinder.getStructure(); TextStructure structure = structureFinder.getStructure();
assertEquals(FileStructure.Format.DELIMITED, structure.getFormat()); assertEquals(TextStructure.Format.DELIMITED, structure.getFormat());
assertEquals(charset, structure.getCharset()); assertEquals(charset, structure.getCharset());
if (hasByteOrderMarker == null) { if (hasByteOrderMarker == null) {
assertNull(structure.getHasByteOrderMarker()); assertNull(structure.getHasByteOrderMarker());
@ -560,9 +560,9 @@ public class DelimitedFileStructureFinderTests extends FileStructureTestCase {
NOOP_TIMEOUT_CHECKER NOOP_TIMEOUT_CHECKER
); );
FileStructure structure = structureFinder.getStructure(); TextStructure structure = structureFinder.getStructure();
assertEquals(FileStructure.Format.DELIMITED, structure.getFormat()); assertEquals(TextStructure.Format.DELIMITED, structure.getFormat());
assertEquals(charset, structure.getCharset()); assertEquals(charset, structure.getCharset());
if (hasByteOrderMarker == null) { if (hasByteOrderMarker == null) {
assertNull(structure.getHasByteOrderMarker()); assertNull(structure.getHasByteOrderMarker());
@ -626,9 +626,9 @@ public class DelimitedFileStructureFinderTests extends FileStructureTestCase {
NOOP_TIMEOUT_CHECKER NOOP_TIMEOUT_CHECKER
); );
FileStructure structure = structureFinder.getStructure(); TextStructure structure = structureFinder.getStructure();
assertEquals(FileStructure.Format.DELIMITED, structure.getFormat()); assertEquals(TextStructure.Format.DELIMITED, structure.getFormat());
assertEquals(charset, structure.getCharset()); assertEquals(charset, structure.getCharset());
if (hasByteOrderMarker == null) { if (hasByteOrderMarker == null) {
assertNull(structure.getHasByteOrderMarker()); assertNull(structure.getHasByteOrderMarker());
@ -676,9 +676,9 @@ public class DelimitedFileStructureFinderTests extends FileStructureTestCase {
NOOP_TIMEOUT_CHECKER NOOP_TIMEOUT_CHECKER
); );
FileStructure structure = structureFinder.getStructure(); TextStructure structure = structureFinder.getStructure();
assertEquals(FileStructure.Format.DELIMITED, structure.getFormat()); assertEquals(TextStructure.Format.DELIMITED, structure.getFormat());
assertEquals(charset, structure.getCharset()); assertEquals(charset, structure.getCharset());
if (hasByteOrderMarker == null) { if (hasByteOrderMarker == null) {
assertNull(structure.getHasByteOrderMarker()); assertNull(structure.getHasByteOrderMarker());
@ -717,9 +717,9 @@ public class DelimitedFileStructureFinderTests extends FileStructureTestCase {
NOOP_TIMEOUT_CHECKER NOOP_TIMEOUT_CHECKER
); );
FileStructure structure = structureFinder.getStructure(); TextStructure structure = structureFinder.getStructure();
assertEquals(FileStructure.Format.DELIMITED, structure.getFormat()); assertEquals(TextStructure.Format.DELIMITED, structure.getFormat());
assertEquals(charset, structure.getCharset()); assertEquals(charset, structure.getCharset());
if (hasByteOrderMarker == null) { if (hasByteOrderMarker == null) {
assertNull(structure.getHasByteOrderMarker()); assertNull(structure.getHasByteOrderMarker());

View file

@ -14,7 +14,7 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
public class FieldStatsCalculatorTests extends FileStructureTestCase { public class FieldStatsCalculatorTests extends TextStructureTestCase {
private static final Map<String, String> LONG = Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "long"); private static final Map<String, String> LONG = Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "long");
private static final Map<String, String> DOUBLE = Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "double"); private static final Map<String, String> DOUBLE = Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "double");

View file

@ -17,7 +17,7 @@ import java.util.Map;
import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsInAnyOrder;
public class GrokPatternCreatorTests extends FileStructureTestCase { public class GrokPatternCreatorTests extends TextStructureTestCase {
public void testBuildFieldName() { public void testBuildFieldName() {
Map<String, Integer> fieldNameCountStore = new HashMap<>(); Map<String, Integer> fieldNameCountStore = new HashMap<>();

View file

@ -5,7 +5,7 @@
*/ */
package org.elasticsearch.xpack.textstructure.structurefinder; package org.elasticsearch.xpack.textstructure.structurefinder;
public class NdJsonFileStructureFinderFactoryTests extends FileStructureTestCase { public class NdJsonTextStructureFinderFactoryTests extends TextStructureTestCase {
private FileStructureFinderFactory factory = new NdJsonFileStructureFinderFactory(); private FileStructureFinderFactory factory = new NdJsonFileStructureFinderFactory();

View file

@ -5,11 +5,11 @@
*/ */
package org.elasticsearch.xpack.textstructure.structurefinder; package org.elasticsearch.xpack.textstructure.structurefinder;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FileStructure; import org.elasticsearch.xpack.core.textstructure.structurefinder.TextStructure;
import java.util.Collections; import java.util.Collections;
public class NdJsonFileStructureFinderTests extends FileStructureTestCase { public class NdJsonTextStructureFinderTests extends TextStructureTestCase {
private final FileStructureFinderFactory factory = new NdJsonFileStructureFinderFactory(); private final FileStructureFinderFactory factory = new NdJsonFileStructureFinderFactory();
@ -28,9 +28,9 @@ public class NdJsonFileStructureFinderTests extends FileStructureTestCase {
NOOP_TIMEOUT_CHECKER NOOP_TIMEOUT_CHECKER
); );
FileStructure structure = structureFinder.getStructure(); TextStructure structure = structureFinder.getStructure();
assertEquals(FileStructure.Format.NDJSON, structure.getFormat()); assertEquals(TextStructure.Format.NDJSON, structure.getFormat());
assertEquals(charset, structure.getCharset()); assertEquals(charset, structure.getCharset());
if (hasByteOrderMarker == null) { if (hasByteOrderMarker == null) {
assertNull(structure.getHasByteOrderMarker()); assertNull(structure.getHasByteOrderMarker());

View file

@ -5,7 +5,7 @@
*/ */
package org.elasticsearch.xpack.textstructure.structurefinder; package org.elasticsearch.xpack.textstructure.structurefinder;
public class TextLogFileStructureFinderFactoryTests extends FileStructureTestCase { public class TextLogTextStructureFinderFactoryTests extends TextStructureTestCase {
private FileStructureFinderFactory factory = new TextLogFileStructureFinderFactory(); private FileStructureFinderFactory factory = new TextLogFileStructureFinderFactory();

View file

@ -7,7 +7,7 @@ package org.elasticsearch.xpack.textstructure.structurefinder;
import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FieldStats; import org.elasticsearch.xpack.core.textstructure.structurefinder.FieldStats;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FileStructure; import org.elasticsearch.xpack.core.textstructure.structurefinder.TextStructure;
import java.util.Collections; import java.util.Collections;
import java.util.Set; import java.util.Set;
@ -16,7 +16,7 @@ import java.util.stream.Collectors;
import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.not;
public class TextLogFileStructureFinderTests extends FileStructureTestCase { public class TextLogTextStructureFinderTests extends TextStructureTestCase {
private final FileStructureFinderFactory factory = new TextLogFileStructureFinderFactory(); private final FileStructureFinderFactory factory = new TextLogFileStructureFinderFactory();
@ -70,9 +70,9 @@ public class TextLogFileStructureFinderTests extends FileStructureTestCase {
NOOP_TIMEOUT_CHECKER NOOP_TIMEOUT_CHECKER
); );
FileStructure structure = structureFinder.getStructure(); TextStructure structure = structureFinder.getStructure();
assertEquals(FileStructure.Format.SEMI_STRUCTURED_TEXT, structure.getFormat()); assertEquals(TextStructure.Format.SEMI_STRUCTURED_TEXT, structure.getFormat());
assertEquals(charset, structure.getCharset()); assertEquals(charset, structure.getCharset());
if (hasByteOrderMarker == null) { if (hasByteOrderMarker == null) {
assertNull(structure.getHasByteOrderMarker()); assertNull(structure.getHasByteOrderMarker());
@ -119,9 +119,9 @@ public class TextLogFileStructureFinderTests extends FileStructureTestCase {
NOOP_TIMEOUT_CHECKER NOOP_TIMEOUT_CHECKER
); );
FileStructure structure = structureFinder.getStructure(); TextStructure structure = structureFinder.getStructure();
assertEquals(FileStructure.Format.SEMI_STRUCTURED_TEXT, structure.getFormat()); assertEquals(TextStructure.Format.SEMI_STRUCTURED_TEXT, structure.getFormat());
assertEquals(charset, structure.getCharset()); assertEquals(charset, structure.getCharset());
if (hasByteOrderMarker == null) { if (hasByteOrderMarker == null) {
assertNull(structure.getHasByteOrderMarker()); assertNull(structure.getHasByteOrderMarker());
@ -163,9 +163,9 @@ public class TextLogFileStructureFinderTests extends FileStructureTestCase {
NOOP_TIMEOUT_CHECKER NOOP_TIMEOUT_CHECKER
); );
FileStructure structure = structureFinder.getStructure(); TextStructure structure = structureFinder.getStructure();
assertEquals(FileStructure.Format.SEMI_STRUCTURED_TEXT, structure.getFormat()); assertEquals(TextStructure.Format.SEMI_STRUCTURED_TEXT, structure.getFormat());
assertEquals(charset, structure.getCharset()); assertEquals(charset, structure.getCharset());
if (hasByteOrderMarker == null) { if (hasByteOrderMarker == null) {
assertNull(structure.getHasByteOrderMarker()); assertNull(structure.getHasByteOrderMarker());
@ -212,9 +212,9 @@ public class TextLogFileStructureFinderTests extends FileStructureTestCase {
NOOP_TIMEOUT_CHECKER NOOP_TIMEOUT_CHECKER
); );
FileStructure structure = structureFinder.getStructure(); TextStructure structure = structureFinder.getStructure();
assertEquals(FileStructure.Format.SEMI_STRUCTURED_TEXT, structure.getFormat()); assertEquals(TextStructure.Format.SEMI_STRUCTURED_TEXT, structure.getFormat());
assertEquals(charset, structure.getCharset()); assertEquals(charset, structure.getCharset());
if (hasByteOrderMarker == null) { if (hasByteOrderMarker == null) {
assertNull(structure.getHasByteOrderMarker()); assertNull(structure.getHasByteOrderMarker());

View file

@ -9,7 +9,7 @@ import com.ibm.icu.text.CharsetMatch;
import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.threadpool.Scheduler; import org.elasticsearch.threadpool.Scheduler;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FileStructure; import org.elasticsearch.xpack.core.textstructure.structurefinder.TextStructure;
import org.junit.After; import org.junit.After;
import org.junit.Before; import org.junit.Before;
@ -32,7 +32,7 @@ import static org.hamcrest.Matchers.endsWith;
import static org.hamcrest.Matchers.startsWith; import static org.hamcrest.Matchers.startsWith;
import static org.hamcrest.core.IsInstanceOf.instanceOf; import static org.hamcrest.core.IsInstanceOf.instanceOf;
public class FileStructureFinderManagerTests extends FileStructureTestCase { public class TextStructureFinderManagerTests extends TextStructureTestCase {
private ScheduledExecutorService scheduler; private ScheduledExecutorService scheduler;
private FileStructureFinderManager structureFinderManager; private FileStructureFinderManager structureFinderManager;
@ -131,7 +131,7 @@ public class FileStructureFinderManagerTests extends FileStructureTestCase {
// Need to change the quote character from the default of double quotes // Need to change the quote character from the default of double quotes
// otherwise the quotes in the NDJSON will stop it parsing as CSV // otherwise the quotes in the NDJSON will stop it parsing as CSV
FileStructureOverrides overrides = FileStructureOverrides.builder() FileStructureOverrides overrides = FileStructureOverrides.builder()
.setFormat(FileStructure.Format.DELIMITED) .setFormat(TextStructure.Format.DELIMITED)
.setQuote('\'') .setQuote('\'')
.build(); .build();
@ -166,7 +166,7 @@ public class FileStructureFinderManagerTests extends FileStructureTestCase {
public void testMakeBestStructureGivenXmlAndTextOverride() throws Exception { public void testMakeBestStructureGivenXmlAndTextOverride() throws Exception {
FileStructureOverrides overrides = FileStructureOverrides.builder().setFormat(FileStructure.Format.SEMI_STRUCTURED_TEXT).build(); FileStructureOverrides overrides = FileStructureOverrides.builder().setFormat(TextStructure.Format.SEMI_STRUCTURED_TEXT).build();
assertThat( assertThat(
structureFinderManager.makeBestStructureFinder( structureFinderManager.makeBestStructureFinder(
@ -199,7 +199,7 @@ public class FileStructureFinderManagerTests extends FileStructureTestCase {
public void testMakeBestStructureGivenCsvAndJsonOverride() { public void testMakeBestStructureGivenCsvAndJsonOverride() {
FileStructureOverrides overrides = FileStructureOverrides.builder().setFormat(FileStructure.Format.NDJSON).build(); FileStructureOverrides overrides = FileStructureOverrides.builder().setFormat(TextStructure.Format.NDJSON).build();
IllegalArgumentException e = expectThrows( IllegalArgumentException e = expectThrows(
IllegalArgumentException.class, IllegalArgumentException.class,
@ -236,7 +236,7 @@ public class FileStructureFinderManagerTests extends FileStructureTestCase {
// Every line of the text sample has two colons, so colon delimited is possible, just very weird // Every line of the text sample has two colons, so colon delimited is possible, just very weird
FileStructureOverrides overrides = FileStructureOverrides.builder() FileStructureOverrides overrides = FileStructureOverrides.builder()
.setFormat(FileStructure.Format.DELIMITED) .setFormat(TextStructure.Format.DELIMITED)
.setDelimiter(':') .setDelimiter(':')
.build(); .build();

View file

@ -17,7 +17,7 @@ import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.stream.Collectors; import java.util.stream.Collectors;
public abstract class FileStructureTestCase extends ESTestCase { public abstract class TextStructureTestCase extends ESTestCase {
protected static final List<String> POSSIBLE_CHARSETS = Collections.unmodifiableList( protected static final List<String> POSSIBLE_CHARSETS = Collections.unmodifiableList(
Charset.availableCharsets() Charset.availableCharsets()

View file

@ -20,7 +20,7 @@ import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.instanceOf;
public class FileStructureUtilsTests extends FileStructureTestCase { public class TextStructureUtilsTests extends TextStructureTestCase {
public void testMoreLikelyGivenText() { public void testMoreLikelyGivenText() {
assertTrue(FileStructureUtils.isMoreLikelyTextThanKeyword("the quick brown fox jumped over the lazy dog")); assertTrue(FileStructureUtils.isMoreLikelyTextThanKeyword("the quick brown fox jumped over the lazy dog"));
@ -482,7 +482,7 @@ public class FileStructureUtilsTests extends FileStructureTestCase {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public void testMakeIngestPipelineDefinitionGivenDelimitedWithoutTimestamp() { public void testMakeIngestPipelineDefinitionGivenDelimitedWithoutTimestamp() {
Map<String, Object> csvProcessorSettings = DelimitedFileStructureFinderTests.randomCsvProcessorSettings(); Map<String, Object> csvProcessorSettings = DelimitedTextStructureFinderTests.randomCsvProcessorSettings();
Map<String, Object> pipeline = FileStructureUtils.makeIngestPipelineDefinition( Map<String, Object> pipeline = FileStructureUtils.makeIngestPipelineDefinition(
null, null,
@ -518,7 +518,7 @@ public class FileStructureUtilsTests extends FileStructureTestCase {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public void testMakeIngestPipelineDefinitionGivenDelimitedWithFieldInTargetFields() { public void testMakeIngestPipelineDefinitionGivenDelimitedWithFieldInTargetFields() {
Map<String, Object> csvProcessorSettings = new HashMap<>(DelimitedFileStructureFinderTests.randomCsvProcessorSettings()); Map<String, Object> csvProcessorSettings = new HashMap<>(DelimitedTextStructureFinderTests.randomCsvProcessorSettings());
// Hack it so the field to be parsed is also one of the column names // Hack it so the field to be parsed is also one of the column names
String firstTargetField = ((List<String>) csvProcessorSettings.get("target_fields")).get(0); String firstTargetField = ((List<String>) csvProcessorSettings.get("target_fields")).get(0);
csvProcessorSettings.put("field", firstTargetField); csvProcessorSettings.put("field", firstTargetField);
@ -554,7 +554,7 @@ public class FileStructureUtilsTests extends FileStructureTestCase {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public void testMakeIngestPipelineDefinitionGivenDelimitedWithConversion() { public void testMakeIngestPipelineDefinitionGivenDelimitedWithConversion() {
Map<String, Object> csvProcessorSettings = DelimitedFileStructureFinderTests.randomCsvProcessorSettings(); Map<String, Object> csvProcessorSettings = DelimitedTextStructureFinderTests.randomCsvProcessorSettings();
boolean expectConversion = randomBoolean(); boolean expectConversion = randomBoolean();
String mappingType = expectConversion ? randomFrom("long", "double", "boolean") : randomFrom("keyword", "text", "date"); String mappingType = expectConversion ? randomFrom("long", "double", "boolean") : randomFrom("keyword", "text", "date");
String firstTargetField = ((List<String>) csvProcessorSettings.get("target_fields")).get(0); String firstTargetField = ((List<String>) csvProcessorSettings.get("target_fields")).get(0);
@ -606,7 +606,7 @@ public class FileStructureUtilsTests extends FileStructureTestCase {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public void testMakeIngestPipelineDefinitionGivenDelimitedWithTimestamp() { public void testMakeIngestPipelineDefinitionGivenDelimitedWithTimestamp() {
Map<String, Object> csvProcessorSettings = DelimitedFileStructureFinderTests.randomCsvProcessorSettings(); Map<String, Object> csvProcessorSettings = DelimitedTextStructureFinderTests.randomCsvProcessorSettings();
String timestampField = randomAlphaOfLength(10); String timestampField = randomAlphaOfLength(10);
List<String> timestampFormats = randomFrom( List<String> timestampFormats = randomFrom(

View file

@ -19,7 +19,7 @@ import static org.hamcrest.Matchers.equalTo;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verify;
public class TimeoutCheckerTests extends FileStructureTestCase { public class TimeoutCheckerTests extends TextStructureTestCase {
private ScheduledExecutorService scheduler; private ScheduledExecutorService scheduler;

View file

@ -23,7 +23,7 @@ import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.regex.Pattern; import java.util.regex.Pattern;
public class TimestampFormatFinderTests extends FileStructureTestCase { public class TimestampFormatFinderTests extends TextStructureTestCase {
private static final String EXCEPTION_TRACE_SAMPLE = private static final String EXCEPTION_TRACE_SAMPLE =
"[2018-02-28T14:49:40,517][DEBUG][o.e.a.b.TransportShardBulkAction] [an_index][2] failed to execute bulk item " "[2018-02-28T14:49:40,517][DEBUG][o.e.a.b.TransportShardBulkAction] [an_index][2] failed to execute bulk item "

View file

@ -5,7 +5,7 @@
*/ */
package org.elasticsearch.xpack.textstructure.structurefinder; package org.elasticsearch.xpack.textstructure.structurefinder;
public class XmlFileStructureFinderFactoryTests extends FileStructureTestCase { public class XmlTextStructureFinderFactoryTests extends TextStructureTestCase {
private FileStructureFinderFactory factory = new XmlFileStructureFinderFactory(); private FileStructureFinderFactory factory = new XmlFileStructureFinderFactory();

View file

@ -5,11 +5,11 @@
*/ */
package org.elasticsearch.xpack.textstructure.structurefinder; package org.elasticsearch.xpack.textstructure.structurefinder;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FileStructure; import org.elasticsearch.xpack.core.textstructure.structurefinder.TextStructure;
import java.util.Collections; import java.util.Collections;
public class XmlFileStructureFinderTests extends FileStructureTestCase { public class XmlTextStructureFinderTests extends TextStructureTestCase {
private final FileStructureFinderFactory factory = new XmlFileStructureFinderFactory(); private final FileStructureFinderFactory factory = new XmlFileStructureFinderFactory();
@ -28,9 +28,9 @@ public class XmlFileStructureFinderTests extends FileStructureTestCase {
NOOP_TIMEOUT_CHECKER NOOP_TIMEOUT_CHECKER
); );
FileStructure structure = structureFinder.getStructure(); TextStructure structure = structureFinder.getStructure();
assertEquals(FileStructure.Format.XML, structure.getFormat()); assertEquals(TextStructure.Format.XML, structure.getFormat());
assertEquals(charset, structure.getCharset()); assertEquals(charset, structure.getCharset());
if (hasByteOrderMarker == null) { if (hasByteOrderMarker == null) {
assertNull(structure.getHasByteOrderMarker()); assertNull(structure.getHasByteOrderMarker());