github-mirrors/elasticsearch
1
0
Fork 0
mirror of https://github.com/elastic/elasticsearch.git synced 2025-04-24 23:27:25 -04:00
Code Issues Projects Releases Packages Wiki Activity

move ignore parameter support from yaml test client to low level rest client (#22637)

Browse source 
All the language clients support a special ignore parameter that doesn't get passed to elasticsearch with the request, but used to indicate which error code should not lead to an exception if returned for a specific request.

Moving this to the low level REST client will allow the high level REST client to make use of it too, for instance so that it doesn't have to intercept ResponseExceptions when the get api returns a 404.
This commit is contained in:
Luca Cavanna 2017-01-16 18:54:44 +01:00 • committed by GitHub
parent eea4db5512
commit 193111919c
5 changed files with 101 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

49
client/rest/src/main/java/org/elasticsearch/client/RestClient.java
View file

@ -49,6 +49,7 @@ import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator; import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
@ -282,15 +283,44 @@ public class RestClient implements Closeable {
public void performRequestAsync(String method, String endpoint, Map<String, String> params, public void performRequestAsync(String method, String endpoint, Map<String, String> params,
HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory,
ResponseListener responseListener, Header... headers) { ResponseListener responseListener, Header... headers) {
URI uri = buildUri(pathPrefix, endpoint, params); Objects.requireNonNull(params, "params must not be null");
Map<String, String> requestParams = new HashMap<>(params);
//ignore is a special parameter supported by the clients, shouldn't be sent to es
String ignoreString = requestParams.remove("ignore");
Set<Integer> ignoreErrorCodes;
if (ignoreString == null) {
if (HttpHead.METHOD_NAME.equals(method)) {
//404 never causes error if returned for a HEAD request
ignoreErrorCodes = Collections.singleton(404);
} else {
ignoreErrorCodes = Collections.emptySet();
}
} else {
String[] ignoresArray = ignoreString.split(",");
ignoreErrorCodes = new HashSet<>();
if (HttpHead.METHOD_NAME.equals(method)) {
//404 never causes error if returned for a HEAD request
ignoreErrorCodes.add(404);
}
for (String ignoreCode : ignoresArray) {
try {
ignoreErrorCodes.add(Integer.valueOf(ignoreCode));
} catch (NumberFormatException e) {
throw new IllegalArgumentException("ignore value should be a number, found [" + ignoreString + "] instead", e);
}
}
}
URI uri = buildUri(pathPrefix, endpoint, requestParams);
HttpRequestBase request = createHttpRequest(method, uri, entity); HttpRequestBase request = createHttpRequest(method, uri, entity);
setHeaders(request, headers); setHeaders(request, headers);
FailureTrackingResponseListener failureTrackingResponseListener = new FailureTrackingResponseListener(responseListener); FailureTrackingResponseListener failureTrackingResponseListener = new FailureTrackingResponseListener(responseListener);
long startTime = System.nanoTime(); long startTime = System.nanoTime();
performRequestAsync(startTime, nextHost().iterator(), request, httpAsyncResponseConsumerFactory, failureTrackingResponseListener); performRequestAsync(startTime, nextHost().iterator(), request, ignoreErrorCodes, httpAsyncResponseConsumerFactory,
failureTrackingResponseListener);
} }
private void performRequestAsync(final long startTime, final Iterator<HttpHost> hosts, final HttpRequestBase request, private void performRequestAsync(final long startTime, final Iterator<HttpHost> hosts, final HttpRequestBase request,
final Set<Integer> ignoreErrorCodes,
final HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, final HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory,
final FailureTrackingResponseListener listener) { final FailureTrackingResponseListener listener) {
final HttpHost host = hosts.next(); final HttpHost host = hosts.next();
@ -304,7 +334,7 @@ public class RestClient implements Closeable {
RequestLogger.logResponse(logger, request, host, httpResponse); RequestLogger.logResponse(logger, request, host, httpResponse);
int statusCode = httpResponse.getStatusLine().getStatusCode(); int statusCode = httpResponse.getStatusLine().getStatusCode();
Response response = new Response(request.getRequestLine(), host, httpResponse); Response response = new Response(request.getRequestLine(), host, httpResponse);
if (isSuccessfulResponse(request.getMethod(), statusCode)) { if (isSuccessfulResponse(statusCode) || ignoreErrorCodes.contains(response.getStatusLine().getStatusCode())) {
onResponse(host); onResponse(host);
listener.onSuccess(response); listener.onSuccess(response);
} else { } else {
@ -312,7 +342,7 @@ public class RestClient implements Closeable {
if (isRetryStatus(statusCode)) { if (isRetryStatus(statusCode)) {
//mark host dead and retry against next one //mark host dead and retry against next one
onFailure(host); onFailure(host);
retryIfPossible(responseException, hosts, request); retryIfPossible(responseException);
} else { } else {
//mark host alive and don't retry, as the error should be a request problem //mark host alive and don't retry, as the error should be a request problem
onResponse(host); onResponse(host);
@ -329,13 +359,13 @@ public class RestClient implements Closeable {
try { try {
RequestLogger.logFailedRequest(logger, request, host, failure); RequestLogger.logFailedRequest(logger, request, host, failure);
onFailure(host); onFailure(host);
retryIfPossible(failure, hosts, request); retryIfPossible(failure);
} catch(Exception e) { } catch(Exception e) {
listener.onDefinitiveFailure(e); listener.onDefinitiveFailure(e);
} }
} }
private void retryIfPossible(Exception exception, Iterator<HttpHost> hosts, HttpRequestBase request) { private void retryIfPossible(Exception exception) {
if (hosts.hasNext()) { if (hosts.hasNext()) {
//in case we are retrying, check whether maxRetryTimeout has been reached //in case we are retrying, check whether maxRetryTimeout has been reached
long timeElapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime); long timeElapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
@ -347,7 +377,7 @@ public class RestClient implements Closeable {
} else { } else {
listener.trackFailure(exception); listener.trackFailure(exception);
request.reset(); request.reset();
performRequestAsync(startTime, hosts, request, httpAsyncResponseConsumerFactory, listener); performRequestAsync(startTime, hosts, request, ignoreErrorCodes, httpAsyncResponseConsumerFactory, listener);
} }
} else { } else {
listener.onDefinitiveFailure(exception); listener.onDefinitiveFailure(exception);
@ -452,8 +482,8 @@ public class RestClient implements Closeable {
client.close(); client.close();
} }
private static boolean isSuccessfulResponse(String method, int statusCode) { private static boolean isSuccessfulResponse(int statusCode) {
return statusCode < 300 || (HttpHead.METHOD_NAME.equals(method) && statusCode == 404); return statusCode < 300;
} }
private static boolean isRetryStatus(int statusCode) { private static boolean isRetryStatus(int statusCode) {
@ -510,7 +540,6 @@ public class RestClient implements Closeable {
} }
private static URI buildUri(String pathPrefix, String path, Map<String, String> params) { private static URI buildUri(String pathPrefix, String path, Map<String, String> params) {
Objects.requireNonNull(params, "params must not be null");
Objects.requireNonNull(path, "path must not be null"); Objects.requireNonNull(path, "path must not be null");
try { try {
String fullPath; String fullPath;

64
client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
View file

@ -216,23 +216,45 @@ public class RestClientSingleHostTests extends RestClientTestCase {
*/ */
public void testErrorStatusCodes() throws IOException { public void testErrorStatusCodes() throws IOException {
for (String method : getHttpMethods()) { for (String method : getHttpMethods()) {
Set<Integer> expectedIgnores = new HashSet<>();
String ignoreParam = "";
if (HttpHead.METHOD_NAME.equals(method)) {
expectedIgnores.add(404);
}
if (randomBoolean()) {
int numIgnores = randomIntBetween(1, 3);
for (int i = 0; i < numIgnores; i++) {
Integer code = randomFrom(getAllErrorStatusCodes());
expectedIgnores.add(code);
ignoreParam += code;
if (i < numIgnores - 1) {
ignoreParam += ",";
}
}
}
//error status codes should cause an exception to be thrown //error status codes should cause an exception to be thrown
for (int errorStatusCode : getAllErrorStatusCodes()) { for (int errorStatusCode : getAllErrorStatusCodes()) {
try { try {
Response response = performRequest(method, "/" + errorStatusCode); Map<String, String> params;
if (method.equals("HEAD") && errorStatusCode == 404) { if (ignoreParam.isEmpty()) {
//no exception gets thrown although we got a 404 params = Collections.emptyMap();
assertThat(response.getStatusLine().getStatusCode(), equalTo(errorStatusCode)); } else {
params = Collections.singletonMap("ignore", ignoreParam);
}
Response response = performRequest(method, "/" + errorStatusCode, params);
if (expectedIgnores.contains(errorStatusCode)) {
//no exception gets thrown although we got an error status code, as it was configured to be ignored
assertEquals(errorStatusCode, response.getStatusLine().getStatusCode());
} else { } else {
fail("request should have failed"); fail("request should have failed");
} }
} catch(ResponseException e) { } catch(ResponseException e) {
if (method.equals("HEAD") && errorStatusCode == 404) { if (expectedIgnores.contains(errorStatusCode)) {
throw e; throw e;
} }
assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(errorStatusCode)); assertEquals(errorStatusCode, e.getResponse().getStatusLine().getStatusCode());
} }
if (errorStatusCode <= 500) { if (errorStatusCode <= 500 || expectedIgnores.contains(errorStatusCode)) {
failureListener.assertNotCalled(); failureListener.assertNotCalled();
} else { } else {
failureListener.assertCalled(httpHost); failureListener.assertCalled(httpHost);
@ -351,11 +373,10 @@ public class RestClientSingleHostTests extends RestClientTestCase {
private HttpUriRequest performRandomRequest(String method) throws Exception { private HttpUriRequest performRandomRequest(String method) throws Exception {
String uriAsString = "/" + randomStatusCode(getRandom()); String uriAsString = "/" + randomStatusCode(getRandom());
URIBuilder uriBuilder = new URIBuilder(uriAsString); URIBuilder uriBuilder = new URIBuilder(uriAsString);
Map<String, String> params = Collections.emptyMap(); final Map<String, String> params = new HashMap<>();
boolean hasParams = randomBoolean(); boolean hasParams = randomBoolean();
if (hasParams) { if (hasParams) {
int numParams = randomIntBetween(1, 3); int numParams = randomIntBetween(1, 3);
params = new HashMap<>(numParams);
for (int i = 0; i < numParams; i++) { for (int i = 0; i < numParams; i++) {
String paramKey = "param-" + i; String paramKey = "param-" + i;
String paramValue = randomAsciiOfLengthBetween(3, 10); String paramValue = randomAsciiOfLengthBetween(3, 10);
@ -363,6 +384,14 @@ public class RestClientSingleHostTests extends RestClientTestCase {
uriBuilder.addParameter(paramKey, paramValue); uriBuilder.addParameter(paramKey, paramValue);
} }
} }
if (randomBoolean()) {
//randomly add some ignore parameter, which doesn't get sent as part of the request
String ignore = Integer.toString(randomFrom(RestClientTestUtil.getAllErrorStatusCodes()));
if (randomBoolean()) {
ignore += "," + Integer.toString(randomFrom(RestClientTestUtil.getAllErrorStatusCodes()));
}
params.put("ignore", ignore);
}
URI uri = uriBuilder.build(); URI uri = uriBuilder.build();
HttpUriRequest request; HttpUriRequest request;
@ -433,16 +462,25 @@ public class RestClientSingleHostTests extends RestClientTestCase {
} }
private Response performRequest(String method, String endpoint, Header... headers) throws IOException { private Response performRequest(String method, String endpoint, Header... headers) throws IOException {
switch(randomIntBetween(0, 2)) { return performRequest(method, endpoint, Collections.<String, String>emptyMap(), headers);
}
private Response performRequest(String method, String endpoint, Map<String, String> params, Header... headers) throws IOException {
int methodSelector;
if (params.isEmpty()) {
methodSelector = randomIntBetween(0, 2);
} else {
methodSelector = randomIntBetween(1, 2);
}
switch(methodSelector) {
case 0: case 0:
return restClient.performRequest(method, endpoint, headers); return restClient.performRequest(method, endpoint, headers);
case 1: case 1:
return restClient.performRequest(method, endpoint, Collections.<String, String>emptyMap(), headers); return restClient.performRequest(method, endpoint, params, headers);
case 2: case 2:
return restClient.performRequest(method, endpoint, Collections.<String, String>emptyMap(), (HttpEntity)null, headers); return restClient.performRequest(method, endpoint, params, (HttpEntity)null, headers);
default: default:
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
} }
} }

2
client/test/src/main/java/org/elasticsearch/client/RestClientTestUtil.java
View file

@ -59,7 +59,7 @@ final class RestClientTestUtil {
} }
static int randomStatusCode(Random random) { static int randomStatusCode(Random random) {
return RandomPicks.randomFrom(random, ALL_ERROR_STATUS_CODES); return RandomPicks.randomFrom(random, ALL_STATUS_CODES);
} }
static int randomOkStatusCode(Random random) { static int randomOkStatusCode(Random random) {

9
docs/java-rest/usage.asciidoc
View file

@ -206,7 +206,14 @@ access to the returned response.
NOTE: A `ResponseException` is **not** thrown for `HEAD` requests that return NOTE: A `ResponseException` is **not** thrown for `HEAD` requests that return
a `404` status code because it is an expected `HEAD` response that simply a `404` status code because it is an expected `HEAD` response that simply
denotes that the resource is not found. All other HTTP methods (e.g., `GET`) denotes that the resource is not found. All other HTTP methods (e.g., `GET`)
throw a `ResponseException` for `404` responses. throw a `ResponseException` for `404` responses unless the `ignore` parameter
contains `404`. `ignore` is a special client parameter that doesn't get sent
to Elasticsearch and contains a comma separated list of error status codes.
It allows to control whether some error status code should be treated as an
expected response rather than as an exception. This is useful for instance
with the get api as it can return `404` when the document is missing, in which
case the response body will not contain an error but rather the usual get api
response, just without the document as it was not found.
=== Example requests === Example requests

28
test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java
View file

@ -40,8 +40,6 @@ import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec;
import java.io.IOException; import java.io.IOException;
import java.net.URI; import java.net.URI;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -59,7 +57,7 @@ public class ClientYamlTestClient {
* Query params that don't need to be declared in the spec, they are supported by default. * Query params that don't need to be declared in the spec, they are supported by default.
*/ */
private static final Set<String> ALWAYS_ACCEPTED_QUERY_STRING_PARAMS = Sets.newHashSet( private static final Set<String> ALWAYS_ACCEPTED_QUERY_STRING_PARAMS = Sets.newHashSet(
"error_trace", "filter_path", "human", "pretty", "source"); "ignore", "error_trace", "human", "filter_path", "pretty", "source");
private final ClientYamlSuiteRestSpec restSpec; private final ClientYamlSuiteRestSpec restSpec;
private final RestClient restClient; private final RestClient restClient;
@ -101,31 +99,12 @@ public class ClientYamlTestClient {
} }
} }
List<Integer> ignores = new ArrayList<>();
Map<String, String> requestParams;
if (params == null) {
requestParams = Collections.emptyMap();
} else {
requestParams = new HashMap<>(params);
if (params.isEmpty() == false) {
//ignore is a special parameter supported by the clients, shouldn't be sent to es
String ignoreString = requestParams.remove("ignore");
if (ignoreString != null) {
try {
ignores.add(Integer.valueOf(ignoreString));
} catch (NumberFormatException e) {
throw new IllegalArgumentException("ignore value should be a number, found [" + ignoreString + "] instead");
}
}
}
}
ClientYamlSuiteRestApi restApi = restApi(apiName); ClientYamlSuiteRestApi restApi = restApi(apiName);
//divide params between ones that go within query string and ones that go within path //divide params between ones that go within query string and ones that go within path
Map<String, String> pathParts = new HashMap<>(); Map<String, String> pathParts = new HashMap<>();
Map<String, String> queryStringParams = new HashMap<>(); Map<String, String> queryStringParams = new HashMap<>();
for (Map.Entry<String, String> entry : requestParams.entrySet()) { for (Map.Entry<String, String> entry : params.entrySet()) {
if (restApi.getPathParts().contains(entry.getKey())) { if (restApi.getPathParts().contains(entry.getKey())) {
pathParts.put(entry.getKey(), entry.getValue()); pathParts.put(entry.getKey(), entry.getValue());
} else { } else {
@ -197,9 +176,6 @@ public class ClientYamlTestClient {
Response response = restClient.performRequest(requestMethod, requestPath, queryStringParams, requestBody, requestHeaders); Response response = restClient.performRequest(requestMethod, requestPath, queryStringParams, requestBody, requestHeaders);
return new ClientYamlTestResponse(response); return new ClientYamlTestResponse(response);
} catch(ResponseException e) { } catch(ResponseException e) {
if (ignores.contains(e.getResponse().getStatusLine().getStatusCode())) {
return new ClientYamlTestResponse(e.getResponse());
}
throw new ClientYamlTestResponseException(e); throw new ClientYamlTestResponseException(e);
} }
} }