mirror of
https://github.com/elastic/elasticsearch.git
synced 2025-04-25 15:47:23 -04:00
f56a0f4b66
Removes `testenv` annotations and related code. These annotations originally let you skip x-pack snippet tests in the docs. However, that's no longer possible. Relates to #79309, #31619
331 lines
10 KiB
Text
331 lines
10 KiB
Text
[role="xpack"]
|
||
[[ccr-getting-started-tutorial]]
|
||
=== Tutorial: Set up {ccr}
|
||
++++
|
||
<titleabbrev>Set up {ccr}</titleabbrev>
|
||
++++
|
||
|
||
////
|
||
[source,console]
|
||
----
|
||
PUT /server-metrics
|
||
{
|
||
"settings" : {
|
||
"index" : {
|
||
"number_of_shards" : 1,
|
||
"number_of_replicas" : 0
|
||
}
|
||
},
|
||
"mappings" : {
|
||
"properties" : {
|
||
"@timestamp" : {
|
||
"type" : "date"
|
||
},
|
||
"accept" : {
|
||
"type" : "long"
|
||
},
|
||
"deny" : {
|
||
"type" : "long"
|
||
},
|
||
"host" : {
|
||
"type" : "keyword"
|
||
},
|
||
"response" : {
|
||
"type" : "float"
|
||
},
|
||
"service" : {
|
||
"type" : "keyword"
|
||
},
|
||
"total" : {
|
||
"type" : "long"
|
||
}
|
||
}
|
||
}
|
||
}
|
||
----
|
||
// TESTSETUP
|
||
////
|
||
|
||
Use this guide to set up {ccr} (CCR) between clusters in two
|
||
datacenters. Replicating your data across datacenters provides several benefits:
|
||
|
||
* Brings data closer to your users or application server to reduce latency and
|
||
response time
|
||
* Provides your mission-critical applications with the tolerance to withstand datacenter or region outages
|
||
|
||
In this guide, you'll learn how to:
|
||
|
||
* Configure a <<remote-clusters,remote cluster>> with a leader index
|
||
* Create a follower index on a local cluster
|
||
* Create an auto-follow pattern to automatically follow time series indices
|
||
that are periodically created in a remote cluster
|
||
|
||
You can manually create follower indices to replicate specific indices on a
|
||
remote cluster, or configure auto-follow patterns to replicate rolling time series indices.
|
||
|
||
TIP: If you want to replicate data across clusters in the cloud, you can
|
||
link:{cloud}/ec-enable-ccs.html[configure remote clusters on {ess}]. Then, you
|
||
can <<modules-cross-cluster-search,search across clusters>> and set up {ccr}.
|
||
|
||
[[ccr-getting-started-prerequisites]]
|
||
==== Prerequisites
|
||
To complete this tutorial, you need:
|
||
|
||
* The `manage` cluster privilege on the local cluster.
|
||
* A license on both clusters that includes {ccr}. {kibana-ref}/managing-licenses.html[Activate a free 30-day trial].
|
||
* An index on the remote cluster that contains the data you want to replicate.
|
||
This tutorial uses the sample eCommerce orders data set.
|
||
{kibana-ref}/get-started.html#gs-get-data-into-kibana[Load sample data].
|
||
* In the local cluster, all nodes with the `master` <<node-roles,node role>> must
|
||
also have the <<remote-node,`remote_cluster_client`>> role. The local cluster
|
||
must also have at least one node with both a data role and the
|
||
<<remote-node,`remote_cluster_client`>> role. Individual tasks for coordinating
|
||
replication scale based on the number of data nodes with the
|
||
`remote_cluster_client` role in the local cluster.
|
||
|
||
==== Connect to a remote cluster
|
||
To replicate an index on a remote cluster (Cluster A) to a local cluster (Cluster B), you configure Cluster A as a remote on Cluster B.
|
||
|
||
image::images/ccr-tutorial-clusters.png[ClusterA contains the leader index and ClusterB contains the follower index]
|
||
|
||
To configure a remote cluster from Stack Management in {kib}:
|
||
|
||
. Select *Remote Clusters* from the side navigation.
|
||
. Specify the {es} endpoint URL, or the IP address or host name of the remote
|
||
cluster (`ClusterA`) followed by the transport port (defaults to `9300`). For
|
||
example, `cluster.es.eastus2.staging.azure.foundit.no:9400` or
|
||
`192.168.1.1:9300`.
|
||
|
||
[%collapsible%open]
|
||
.API example
|
||
====
|
||
You can also use the <<cluster-update-settings,cluster update settings API>> to
|
||
add a remote cluster:
|
||
|
||
[source,console]
|
||
----
|
||
PUT /_cluster/settings
|
||
{
|
||
"persistent" : {
|
||
"cluster" : {
|
||
"remote" : {
|
||
"leader" : {
|
||
"seeds" : [
|
||
"127.0.0.1:9300" <1>
|
||
]
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
----
|
||
// TEST[setup:host]
|
||
// TEST[s/127.0.0.1:9300/\${transport_host}/]
|
||
<1> Specifies the hostname and transport port of a seed node in the remote
|
||
cluster.
|
||
|
||
You can verify that the local cluster is successfully connected to the remote
|
||
cluster.
|
||
|
||
[source,console]
|
||
----
|
||
GET /_remote/info
|
||
----
|
||
// TEST[continued]
|
||
|
||
The API response indicates that the local cluster is connected to the remote
|
||
cluster with cluster alias `leader`.
|
||
|
||
[source,console-result]
|
||
----
|
||
{
|
||
"leader" : {
|
||
"seeds" : [
|
||
"127.0.0.1:9300"
|
||
],
|
||
"connected" : true,
|
||
"num_nodes_connected" : 1, <1>
|
||
"max_connections_per_cluster" : 3,
|
||
"initial_connect_timeout" : "30s",
|
||
"skip_unavailable" : false,
|
||
"mode" : "sniff"
|
||
}
|
||
}
|
||
----
|
||
// TESTRESPONSE[s/127.0.0.1:9300/$body.leader.seeds.0/]
|
||
// TEST[s/"connected" : true/"connected" : $body.leader.connected/]
|
||
// TEST[s/"num_nodes_connected" : 1/"num_nodes_connected" : $body.leader.num_nodes_connected/]
|
||
<1> The number of nodes in the remote cluster the local cluster is
|
||
connected to.
|
||
====
|
||
|
||
[[ccr-enable-soft-deletes]]
|
||
==== Enable soft deletes on leader indices
|
||
To follow an index, it must have been created with
|
||
<<ccr-leader-requirements,soft deletes>> enabled. If the index doesn’t have
|
||
soft deletes enabled, you must reindex it and use the new index as the leader
|
||
index. Soft deletes are enabled by default on new indices
|
||
created with {es} 7.0.0 and later.
|
||
|
||
include::../../../x-pack/docs/en/security/authentication/remote-clusters-privileges.asciidoc[tag=configure-ccr-privileges]
|
||
|
||
[[ccr-getting-started-follower-index]]
|
||
==== Create a follower index to replicate a specific index
|
||
When you create a follower index, you reference the remote cluster and the
|
||
leader index in your remote cluster.
|
||
|
||
To create a follower index from Stack Management in {kib}:
|
||
|
||
. Select *Cross-Cluster Replication* in the side navigation and choose the
|
||
*Follower Indices* tab.
|
||
. Choose the cluster (ClusterA) containing the leader index you want to
|
||
replicate.
|
||
. Enter the name of the leader index, which is
|
||
`kibana_sample_data_ecommerce` if you are following the tutorial.
|
||
. Enter a name for your follower index, such as `follower-kibana-sample-data`.
|
||
|
||
{es} initializes the follower using the
|
||
<<ccr-remote-recovery, remote recovery>>
|
||
process, which transfers the existing Lucene segment files from the leader
|
||
index to the follower index. The index status changes to *Paused*. When the
|
||
remote recovery process is complete, the index following begins and the status
|
||
changes to *Active*.
|
||
|
||
When you index documents into your leader index, {es} replicates the documents
|
||
in the follower index.
|
||
|
||
[role="screenshot"]
|
||
image::images/ccr-follower-index.png["The Cross-Cluster Replication page in {kib}"]
|
||
|
||
[%collapsible%open]
|
||
.API example
|
||
====
|
||
You can also use the <<ccr-put-follow,create follower API>> to create follower
|
||
indices. When you create a follower index, you must reference the remote cluster
|
||
and the leader index that you created in the remote cluster.
|
||
|
||
When initiating the follower request, the response returns before the
|
||
<<ccr-remote-recovery, remote recovery>> process completes. To wait for the process
|
||
to complete, add the `wait_for_active_shards` parameter to your request.
|
||
|
||
[source,console]
|
||
----
|
||
PUT /server-metrics-follower/_ccr/follow?wait_for_active_shards=1
|
||
{
|
||
"remote_cluster" : "leader",
|
||
"leader_index" : "server-metrics"
|
||
}
|
||
----
|
||
// TEST[continued]
|
||
|
||
//////////////////////////
|
||
|
||
[source,console-result]
|
||
--------------------------------------------------
|
||
{
|
||
"follow_index_created" : true,
|
||
"follow_index_shards_acked" : true,
|
||
"index_following_started" : true
|
||
}
|
||
--------------------------------------------------
|
||
|
||
//////////////////////////
|
||
|
||
Use the
|
||
<<ccr-get-follow-stats,get follower stats API>> to inspect the status of
|
||
replication.
|
||
|
||
//////////////////////////
|
||
|
||
[source,console]
|
||
--------------------------------------------------
|
||
POST /server-metrics-follower/_ccr/pause_follow
|
||
|
||
POST /server-metrics-follower/_close
|
||
|
||
POST /server-metrics-follower/_ccr/unfollow
|
||
--------------------------------------------------
|
||
// TEST[continued]
|
||
|
||
//////////////////////////
|
||
====
|
||
|
||
[[ccr-getting-started-auto-follow]]
|
||
==== Create an auto-follow pattern to replicate time series indices
|
||
You use <<ccr-auto-follow,auto-follow patterns>> to automatically create new
|
||
followers for rolling time series indices. Whenever the name of a new index on
|
||
the remote cluster matches the auto-follow pattern, a corresponding follower
|
||
index is added to the local cluster.
|
||
|
||
An auto-follow pattern specifies the remote cluster you want to replicate from,
|
||
and one or more index patterns that specify the rolling time series indices you
|
||
want to replicate.
|
||
|
||
// tag::ccr-create-auto-follow-pattern-tag[]
|
||
To create an auto-follow pattern from Stack Management in {kib}:
|
||
|
||
. Select *Cross Cluster Replication* in the side navigation and choose the
|
||
*Auto-follow patterns* tab.
|
||
. Enter a name for the auto-follow pattern, such as `beats`.
|
||
. Choose the remote cluster that contains the index you want to replicate,
|
||
which in the example scenario is Cluster A.
|
||
. Enter one or more index patterns that identify the indices you want to
|
||
replicate from the remote cluster. For example, enter
|
||
`metricbeat-* packetbeat-*` to automatically create followers for {metricbeat} and {packetbeat} indices.
|
||
. Enter *follower-* as the prefix to apply to the names of the follower indices so
|
||
you can more easily identify replicated indices.
|
||
|
||
As new indices matching these patterns are
|
||
created on the remote, {es} automatically replicates them to local follower indices.
|
||
|
||
[role="screenshot"]
|
||
image::images/auto-follow-patterns.png["The Auto-follow patterns page in {kib}"]
|
||
|
||
// end::ccr-create-auto-follow-pattern-tag[]
|
||
|
||
[%collapsible%open]
|
||
.API example
|
||
====
|
||
Use the <<ccr-put-auto-follow-pattern,create auto-follow pattern API>> to
|
||
configure auto-follow patterns.
|
||
|
||
[source,console]
|
||
----
|
||
PUT /_ccr/auto_follow/beats
|
||
{
|
||
"remote_cluster" : "leader",
|
||
"leader_index_patterns" :
|
||
[
|
||
"metricbeat-*", <1>
|
||
"packetbeat-*" <2>
|
||
],
|
||
"follow_index_pattern" : "{{leader_index}}-copy" <3>
|
||
}
|
||
----
|
||
// TEST[continued]
|
||
<1> Automatically follow new {metricbeat} indices.
|
||
<2> Automatically follow new {packetbeat} indices.
|
||
<3> The name of the follower index is derived from the name of the leader index
|
||
by adding the suffix `-copy` to the name of the leader index.
|
||
|
||
//////////////////////////
|
||
|
||
[source,console-result]
|
||
--------------------------------------------------
|
||
{
|
||
"acknowledged" : true
|
||
}
|
||
--------------------------------------------------
|
||
|
||
//////////////////////////
|
||
|
||
//////////////////////////
|
||
|
||
[source,console]
|
||
--------------------------------------------------
|
||
DELETE /_ccr/auto_follow/beats
|
||
--------------------------------------------------
|
||
// TEST[continued]
|
||
|
||
//////////////////////////
|
||
====
|