mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 17:28:26 -04:00
Added annotation infrastructure + a implementation for shard events
This commit is contained in:
Boaz Leskes
parent
4dcf242cd4
commit
1329321031
7 changed files with 784 additions and 64 deletions
|
|
@ -7,7 +7,7 @@
|
|||
"0": {
|
||||
"id": 0,
|
||||
"type": "topN",
|
||||
"query": "_type:node_stats",
|
||||
"query": "*",
|
||||
"alias": "",
|
||||
"color": "#7EB26D",
|
||||
"pin": false,
|
||||
|
|
@ -23,13 +23,14 @@
|
|||
},
|
||||
"filter": {
|
||||
"idQueue": [
|
||||
2
|
||||
2,
|
||||
3
|
||||
],
|
||||
"list": {
|
||||
"0": {
|
||||
"type": "time",
|
||||
"field": "@timestamp",
|
||||
"from": "now-1h",
|
||||
"from": "now-15m",
|
||||
"to": "now",
|
||||
"mandate": "must",
|
||||
"active": true,
|
||||
|
|
@ -46,8 +47,8 @@
|
|||
}
|
||||
},
|
||||
"ids": [
|
||||
0,
|
||||
1
|
||||
1,
|
||||
0
|
||||
]
|
||||
}
|
||||
},
|
||||
|
|
@ -75,7 +76,7 @@
|
|||
"value_field": "os.cpu.user",
|
||||
"auto_int": true,
|
||||
"resolution": 20,
|
||||
"interval": "1m",
|
||||
"interval": "30s",
|
||||
"intervals": [
|
||||
"auto",
|
||||
"1s",
|
||||
|
|
@ -120,6 +121,16 @@
|
|||
"grid": {
|
||||
"max": null,
|
||||
"min": 0
|
||||
},
|
||||
"annotate": {
|
||||
"enable": false,
|
||||
"query": "*",
|
||||
"size": 20,
|
||||
"field": "_type",
|
||||
"sort": [
|
||||
"_score",
|
||||
"desc"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
@ -138,7 +149,7 @@
|
|||
"value_field": "os.mem.used_percent",
|
||||
"auto_int": true,
|
||||
"resolution": 20,
|
||||
"interval": "5m",
|
||||
"interval": "30s",
|
||||
"intervals": [
|
||||
"auto",
|
||||
"1s",
|
||||
|
|
@ -183,6 +194,16 @@
|
|||
"grid": {
|
||||
"max": 100,
|
||||
"min": 0
|
||||
},
|
||||
"annotate": {
|
||||
"enable": false,
|
||||
"query": "*",
|
||||
"size": 20,
|
||||
"field": "_type",
|
||||
"sort": [
|
||||
"_score",
|
||||
"desc"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
@ -201,7 +222,7 @@
|
|||
"value_field": "os.load_average.1m",
|
||||
"auto_int": true,
|
||||
"resolution": 20,
|
||||
"interval": "1m",
|
||||
"interval": "30s",
|
||||
"intervals": [
|
||||
"auto",
|
||||
"1s",
|
||||
|
|
@ -246,6 +267,16 @@
|
|||
"grid": {
|
||||
"max": null,
|
||||
"min": 0
|
||||
},
|
||||
"annotate": {
|
||||
"enable": false,
|
||||
"query": "*",
|
||||
"size": 20,
|
||||
"field": "_type",
|
||||
"sort": [
|
||||
"_score",
|
||||
"desc"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -345,7 +376,7 @@
|
|||
"value_field": "jvm.mem.heap_used_in_bytes",
|
||||
"auto_int": true,
|
||||
"resolution": 20,
|
||||
"interval": "1m",
|
||||
"interval": "30s",
|
||||
"intervals": [
|
||||
"auto",
|
||||
"1s",
|
||||
|
|
@ -390,6 +421,16 @@
|
|||
"grid": {
|
||||
"max": null,
|
||||
"min": 0
|
||||
},
|
||||
"annotate": {
|
||||
"enable": true,
|
||||
"query": "_type:shard_event AND event:STARTED",
|
||||
"size": 20,
|
||||
"field": "message",
|
||||
"sort": [
|
||||
"_score",
|
||||
"desc"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
@ -408,7 +449,7 @@
|
|||
"value_field": "jvm.gc.collectors.ParNew.collection_time_in_millis",
|
||||
"auto_int": true,
|
||||
"resolution": 20,
|
||||
"interval": "1m",
|
||||
"interval": "30s",
|
||||
"intervals": [
|
||||
"auto",
|
||||
"1s",
|
||||
|
|
@ -453,6 +494,16 @@
|
|||
"grid": {
|
||||
"max": null,
|
||||
"min": 0
|
||||
},
|
||||
"annotate": {
|
||||
"enable": false,
|
||||
"query": "*",
|
||||
"size": 20,
|
||||
"field": "_type",
|
||||
"sort": [
|
||||
"_score",
|
||||
"desc"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
@ -471,7 +522,7 @@
|
|||
"value_field": "jvm.gc.collectors.ParNew.collection_count",
|
||||
"auto_int": true,
|
||||
"resolution": 20,
|
||||
"interval": "1m",
|
||||
"interval": "30s",
|
||||
"intervals": [
|
||||
"auto",
|
||||
"1s",
|
||||
|
|
@ -516,6 +567,16 @@
|
|||
"grid": {
|
||||
"max": null,
|
||||
"min": 0
|
||||
},
|
||||
"annotate": {
|
||||
"enable": false,
|
||||
"query": "*",
|
||||
"size": 20,
|
||||
"field": "_type",
|
||||
"sort": [
|
||||
"_score",
|
||||
"desc"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -1417,7 +1478,7 @@
|
|||
"value_field": "indices.indexing.index_total",
|
||||
"auto_int": true,
|
||||
"resolution": 20,
|
||||
"interval": "1m",
|
||||
"interval": "30s",
|
||||
"intervals": [
|
||||
"auto",
|
||||
"1s",
|
||||
|
|
@ -1460,7 +1521,17 @@
|
|||
"max": null,
|
||||
"min": 0
|
||||
},
|
||||
"title": "Indexing requests"
|
||||
"title": "Indexing requests",
|
||||
"annotate": {
|
||||
"enable": true,
|
||||
"query": "_type:shard_event",
|
||||
"size": 60,
|
||||
"field": "message",
|
||||
"sort": [
|
||||
"_score",
|
||||
"desc"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"span": 4,
|
||||
|
|
@ -1478,7 +1549,7 @@
|
|||
"value_field": "indices.merges.current_size_in_bytes",
|
||||
"auto_int": true,
|
||||
"resolution": 20,
|
||||
"interval": "1m",
|
||||
"interval": "30s",
|
||||
"intervals": [
|
||||
"auto",
|
||||
"1s",
|
||||
|
|
@ -1523,6 +1594,16 @@
|
|||
"grid": {
|
||||
"max": null,
|
||||
"min": 0
|
||||
},
|
||||
"annotate": {
|
||||
"enable": false,
|
||||
"query": "*",
|
||||
"size": 20,
|
||||
"field": "_type",
|
||||
"sort": [
|
||||
"_score",
|
||||
"desc"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
@ -1541,7 +1622,7 @@
|
|||
"value_field": "indices.refresh.total_time_in_millis",
|
||||
"auto_int": true,
|
||||
"resolution": 20,
|
||||
"interval": "1m",
|
||||
"interval": "30s",
|
||||
"intervals": [
|
||||
"auto",
|
||||
"1s",
|
||||
|
|
@ -1584,7 +1665,17 @@
|
|||
"max": null,
|
||||
"min": 0
|
||||
},
|
||||
"title": "Avg refresh time"
|
||||
"title": "Avg refresh time",
|
||||
"annotate": {
|
||||
"enable": false,
|
||||
"query": "*",
|
||||
"size": 20,
|
||||
"field": "_type",
|
||||
"sort": [
|
||||
"_score",
|
||||
"desc"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"notice": false
|
||||
|
|
|
|||
350
dashboards/overview.json
Normal file
350
dashboards/overview.json
Normal file
|
|
@ -0,0 +1,350 @@
|
|||
{
|
||||
"title": "Overview",
|
||||
"services": {
|
||||
"query": {
|
||||
"idQueue": [
|
||||
1,
|
||||
2,
|
||||
3,
|
||||
4
|
||||
],
|
||||
"list": {
|
||||
"0": {
|
||||
"query": "*",
|
||||
"alias": "",
|
||||
"color": "#7EB26D",
|
||||
"id": 0,
|
||||
"pin": false,
|
||||
"type": "lucene",
|
||||
"enable": true
|
||||
}
|
||||
},
|
||||
"ids": [
|
||||
0
|
||||
]
|
||||
},
|
||||
"filter": {
|
||||
"idQueue": [
|
||||
1,
|
||||
2
|
||||
],
|
||||
"list": {
|
||||
"0": {
|
||||
"type": "time",
|
||||
"field": "@timestamp",
|
||||
"from": "now-1h",
|
||||
"to": "now",
|
||||
"mandate": "must",
|
||||
"active": true,
|
||||
"alias": "",
|
||||
"id": 0
|
||||
}
|
||||
},
|
||||
"ids": [
|
||||
0
|
||||
]
|
||||
}
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"title": "Data",
|
||||
"height": "150px",
|
||||
"editable": true,
|
||||
"collapse": false,
|
||||
"collapsable": true,
|
||||
"panels": [
|
||||
{
|
||||
"span": 4,
|
||||
"editable": true,
|
||||
"type": "histogram",
|
||||
"loadingEditor": false,
|
||||
"mode": "max",
|
||||
"time_field": "@timestamp",
|
||||
"queries": {
|
||||
"mode": "all",
|
||||
"ids": [
|
||||
0
|
||||
]
|
||||
},
|
||||
"value_field": "indices_stats.primaries.docs.count",
|
||||
"auto_int": true,
|
||||
"resolution": 30,
|
||||
"interval": "1m",
|
||||
"intervals": [
|
||||
"auto",
|
||||
"1s",
|
||||
"1m",
|
||||
"5m",
|
||||
"10m",
|
||||
"30m",
|
||||
"1h",
|
||||
"3h",
|
||||
"12h",
|
||||
"1d",
|
||||
"1w",
|
||||
"1y"
|
||||
],
|
||||
"fill": 0,
|
||||
"linewidth": 3,
|
||||
"timezone": "browser",
|
||||
"spyable": true,
|
||||
"zoomlinks": true,
|
||||
"bars": false,
|
||||
"stack": false,
|
||||
"points": false,
|
||||
"lines": true,
|
||||
"legend": true,
|
||||
"show_query": true,
|
||||
"legend_counts": false,
|
||||
"x-axis": true,
|
||||
"y-axis": true,
|
||||
"percentage": false,
|
||||
"zerofill": true,
|
||||
"interactive": true,
|
||||
"options": true,
|
||||
"derivative": false,
|
||||
"scale": 1,
|
||||
"tooltip": {
|
||||
"value_type": "cumulative",
|
||||
"query_as_alias": true
|
||||
},
|
||||
"grid": {
|
||||
"max": null,
|
||||
"min": 0
|
||||
},
|
||||
"title": "Document Count",
|
||||
"annotate": {
|
||||
"enable": false,
|
||||
"query": "*",
|
||||
"size": 20,
|
||||
"field": "_type",
|
||||
"sort": [
|
||||
"_score",
|
||||
"desc"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"span": 4,
|
||||
"editable": true,
|
||||
"type": "histogram",
|
||||
"loadingEditor": false,
|
||||
"mode": "max",
|
||||
"time_field": "@timestamp",
|
||||
"queries": {
|
||||
"mode": "all",
|
||||
"ids": [
|
||||
0
|
||||
]
|
||||
},
|
||||
"value_field": "indices_stats.total.search.query_total",
|
||||
"auto_int": true,
|
||||
"resolution": 30,
|
||||
"interval": "1m",
|
||||
"intervals": [
|
||||
"auto",
|
||||
"1s",
|
||||
"1m",
|
||||
"5m",
|
||||
"10m",
|
||||
"30m",
|
||||
"1h",
|
||||
"3h",
|
||||
"12h",
|
||||
"1d",
|
||||
"1w",
|
||||
"1y"
|
||||
],
|
||||
"fill": 0,
|
||||
"linewidth": 3,
|
||||
"timezone": "browser",
|
||||
"spyable": true,
|
||||
"zoomlinks": true,
|
||||
"bars": false,
|
||||
"stack": false,
|
||||
"points": false,
|
||||
"lines": true,
|
||||
"legend": true,
|
||||
"show_query": true,
|
||||
"legend_counts": false,
|
||||
"x-axis": true,
|
||||
"y-axis": true,
|
||||
"percentage": false,
|
||||
"zerofill": true,
|
||||
"interactive": true,
|
||||
"options": true,
|
||||
"derivative": true,
|
||||
"scale": 1,
|
||||
"tooltip": {
|
||||
"value_type": "cumulative",
|
||||
"query_as_alias": true
|
||||
},
|
||||
"grid": {
|
||||
"max": null,
|
||||
"min": 0
|
||||
},
|
||||
"title": "Search requests",
|
||||
"annotate": {
|
||||
"enable": false,
|
||||
"query": "*",
|
||||
"size": 20,
|
||||
"field": "_type",
|
||||
"sort": [
|
||||
"_score",
|
||||
"desc"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"span": 4,
|
||||
"editable": true,
|
||||
"type": "histogram",
|
||||
"loadingEditor": false,
|
||||
"mode": "max",
|
||||
"time_field": "@timestamp",
|
||||
"queries": {
|
||||
"mode": "all",
|
||||
"ids": [
|
||||
0
|
||||
]
|
||||
},
|
||||
"value_field": "indices_stats.primaries.indexing.index_total",
|
||||
"auto_int": true,
|
||||
"resolution": 30,
|
||||
"interval": "1m",
|
||||
"intervals": [
|
||||
"auto",
|
||||
"1s",
|
||||
"1m",
|
||||
"5m",
|
||||
"10m",
|
||||
"30m",
|
||||
"1h",
|
||||
"3h",
|
||||
"12h",
|
||||
"1d",
|
||||
"1w",
|
||||
"1y"
|
||||
],
|
||||
"fill": 0,
|
||||
"linewidth": 3,
|
||||
"timezone": "browser",
|
||||
"spyable": true,
|
||||
"zoomlinks": true,
|
||||
"bars": false,
|
||||
"stack": false,
|
||||
"points": false,
|
||||
"lines": true,
|
||||
"legend": true,
|
||||
"show_query": true,
|
||||
"legend_counts": false,
|
||||
"x-axis": true,
|
||||
"y-axis": true,
|
||||
"percentage": false,
|
||||
"zerofill": true,
|
||||
"interactive": true,
|
||||
"options": true,
|
||||
"derivative": true,
|
||||
"scale": 1,
|
||||
"tooltip": {
|
||||
"value_type": "cumulative",
|
||||
"query_as_alias": true
|
||||
},
|
||||
"grid": {
|
||||
"max": null,
|
||||
"min": 0
|
||||
},
|
||||
"title": "Indexing requests",
|
||||
"annotate": {
|
||||
"enable": false,
|
||||
"query": "*",
|
||||
"size": 20,
|
||||
"field": "_type",
|
||||
"sort": [
|
||||
"_score",
|
||||
"desc"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"notice": false
|
||||
}
|
||||
],
|
||||
"editable": true,
|
||||
"failover": false,
|
||||
"index": {
|
||||
"interval": "day",
|
||||
"pattern": "[es_monitor-]YYYY.MM.DD",
|
||||
"default": "_all"
|
||||
},
|
||||
"style": "dark",
|
||||
"panel_hints": true,
|
||||
"pulldowns": [
|
||||
{
|
||||
"type": "query",
|
||||
"collapse": true,
|
||||
"notice": false,
|
||||
"enable": true,
|
||||
"query": "*",
|
||||
"pinned": true,
|
||||
"history": [],
|
||||
"remember": 10
|
||||
},
|
||||
{
|
||||
"type": "filtering",
|
||||
"collapse": true,
|
||||
"notice": true,
|
||||
"enable": true
|
||||
}
|
||||
],
|
||||
"nav": [
|
||||
{
|
||||
"type": "timepicker",
|
||||
"collapse": false,
|
||||
"notice": false,
|
||||
"enable": true,
|
||||
"status": "Stable",
|
||||
"time_options": [
|
||||
"5m",
|
||||
"15m",
|
||||
"1h",
|
||||
"6h",
|
||||
"12h",
|
||||
"24h",
|
||||
"2d",
|
||||
"7d",
|
||||
"30d"
|
||||
],
|
||||
"refresh_intervals": [
|
||||
"5s",
|
||||
"10s",
|
||||
"30s",
|
||||
"1m",
|
||||
"5m",
|
||||
"15m",
|
||||
"30m",
|
||||
"1h",
|
||||
"2h",
|
||||
"1d"
|
||||
],
|
||||
"timefield": "@timestamp",
|
||||
"now": true,
|
||||
"filter_id": 0
|
||||
}
|
||||
],
|
||||
"loader": {
|
||||
"save_gist": false,
|
||||
"save_elasticsearch": true,
|
||||
"save_local": true,
|
||||
"save_default": true,
|
||||
"save_temp": true,
|
||||
"save_temp_ttl_enable": true,
|
||||
"save_temp_ttl": "30d",
|
||||
"load_gist": false,
|
||||
"load_elasticsearch": true,
|
||||
"load_elasticsearch_size": 20,
|
||||
"load_local": true,
|
||||
"hide": false
|
||||
},
|
||||
"refresh": false
|
||||
}
|
||||
|
|
@ -26,20 +26,30 @@ import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
|
|||
import org.elasticsearch.action.admin.indices.stats.ShardStats;
|
||||
import org.elasticsearch.client.Client;
|
||||
import org.elasticsearch.cluster.ClusterService;
|
||||
import org.elasticsearch.common.Nullable;
|
||||
import org.elasticsearch.common.collect.ImmutableSet;
|
||||
import org.elasticsearch.common.component.AbstractLifecycleComponent;
|
||||
import org.elasticsearch.common.inject.Inject;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
|
||||
import org.elasticsearch.common.util.concurrent.EsExecutors;
|
||||
import org.elasticsearch.discovery.Discovery;
|
||||
import org.elasticsearch.index.shard.ShardId;
|
||||
import org.elasticsearch.index.shard.service.IndexShard;
|
||||
import org.elasticsearch.indices.IndicesLifecycle;
|
||||
import org.elasticsearch.marvel.monitor.annotation.Annotation;
|
||||
import org.elasticsearch.marvel.monitor.annotation.ShardEventAnnotation;
|
||||
import org.elasticsearch.marvel.monitor.exporter.ESExporter;
|
||||
import org.elasticsearch.marvel.monitor.exporter.StatsExporter;
|
||||
import org.elasticsearch.indices.IndicesService;
|
||||
import org.elasticsearch.indices.InternalIndicesService;
|
||||
import org.elasticsearch.node.service.NodeService;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.LinkedBlockingDeque;
|
||||
|
||||
public class StatsExportersService extends AbstractLifecycleComponent<StatsExportersService> {
|
||||
|
||||
|
|
@ -48,12 +58,16 @@ public class StatsExportersService extends AbstractLifecycleComponent<StatsExpor
|
|||
private final ClusterService clusterService;
|
||||
private final Client client;
|
||||
|
||||
private final IndicesLifecycle.Listener indicesLifeCycleListener;
|
||||
|
||||
private volatile ExportingWorker exp;
|
||||
private volatile Thread thread;
|
||||
private final TimeValue interval;
|
||||
|
||||
private Collection<StatsExporter> exporters;
|
||||
|
||||
private final BlockingQueue<Annotation> pendingAnnotationsQueue;
|
||||
|
||||
@Inject
|
||||
public StatsExportersService(Settings settings, IndicesService indicesService,
|
||||
NodeService nodeService, ClusterService clusterService,
|
||||
|
|
@ -68,6 +82,9 @@ public class StatsExportersService extends AbstractLifecycleComponent<StatsExpor
|
|||
|
||||
StatsExporter esExporter = new ESExporter(settings.getComponentSettings(ESExporter.class), discovery);
|
||||
this.exporters = ImmutableSet.of(esExporter);
|
||||
|
||||
indicesLifeCycleListener = new IndicesLifeCycleListener();
|
||||
pendingAnnotationsQueue = ConcurrentCollections.newBlockingQueue();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -79,14 +96,24 @@ public class StatsExportersService extends AbstractLifecycleComponent<StatsExpor
|
|||
this.thread = new Thread(exp, EsExecutors.threadName(settings, "monitor"));
|
||||
this.thread.setDaemon(true);
|
||||
this.thread.start();
|
||||
|
||||
indicesService.indicesLifecycle().addListener(indicesLifeCycleListener);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doStop() throws ElasticSearchException {
|
||||
this.exp.closed = true;
|
||||
this.thread.interrupt();
|
||||
try {
|
||||
this.thread.join(60000);
|
||||
} catch (InterruptedException e) {
|
||||
// we don't care...
|
||||
}
|
||||
for (StatsExporter e : exporters)
|
||||
e.stop();
|
||||
|
||||
indicesService.indicesLifecycle().removeListener(indicesLifeCycleListener);
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -111,52 +138,106 @@ public class StatsExportersService extends AbstractLifecycleComponent<StatsExpor
|
|||
|
||||
// do the actual export..., go over the actual exporters list and...
|
||||
try {
|
||||
logger.debug("Collecting node stats");
|
||||
NodeStats nodeStats = nodeService.stats();
|
||||
exportNodeStats();
|
||||
|
||||
logger.debug("Exporting node stats");
|
||||
for (StatsExporter e : exporters) {
|
||||
try {
|
||||
e.exportNodeStats(nodeStats);
|
||||
} catch (Throwable t) {
|
||||
logger.error("StatsExporter [{}] has thrown an exception:", t, e.name());
|
||||
}
|
||||
}
|
||||
|
||||
logger.debug("Collecting shard stats");
|
||||
ShardStats[] shardStatsArray = indicesService.shardStats(CommonStatsFlags.ALL);
|
||||
|
||||
logger.debug("Exporting shards stats");
|
||||
for (StatsExporter e : exporters) {
|
||||
try {
|
||||
e.exportShardStats(shardStatsArray);
|
||||
} catch (Throwable t) {
|
||||
logger.error("StatsExporter [{}] has thrown an exception:", t, e.name());
|
||||
}
|
||||
}
|
||||
exportShardStats();
|
||||
|
||||
exportAnnotations();
|
||||
|
||||
if (clusterService.state().nodes().localNodeMaster()) {
|
||||
logger.debug("local node is master, exporting aggregated stats");
|
||||
IndicesStatsResponse indicesStatsResponse = client.admin().indices().prepareStats().all().get();
|
||||
for (StatsExporter e : exporters) {
|
||||
try {
|
||||
e.exportIndicesStats(indicesStatsResponse);
|
||||
} catch (Throwable t) {
|
||||
logger.error("StatsExporter [{}] has thrown an exception:", t, e.name());
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
exportIndicesStats();
|
||||
}
|
||||
|
||||
|
||||
} catch (Throwable t) {
|
||||
logger.error("Background thread had an uncaught exception:", t);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
logger.debug("shutting down worker, exporting pending annotation");
|
||||
exportAnnotations();
|
||||
|
||||
logger.debug("worker shutdown");
|
||||
}
|
||||
|
||||
private void exportIndicesStats() {
|
||||
logger.debug("local node is master, exporting aggregated stats");
|
||||
IndicesStatsResponse indicesStatsResponse = client.admin().indices().prepareStats().all().get();
|
||||
for (StatsExporter e : exporters) {
|
||||
try {
|
||||
e.exportIndicesStats(indicesStatsResponse);
|
||||
} catch (Throwable t) {
|
||||
logger.error("StatsExporter [{}] has thrown an exception:", t, e.name());
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
private void exportAnnotations() {
|
||||
logger.debug("Exporting annotations");
|
||||
ArrayList<Annotation> annotationsList = new ArrayList<Annotation>(pendingAnnotationsQueue.size());
|
||||
pendingAnnotationsQueue.drainTo(annotationsList);
|
||||
Annotation[] annotations = new Annotation[annotationsList.size()];
|
||||
annotationsList.toArray(annotations);
|
||||
|
||||
for (StatsExporter e : exporters) {
|
||||
try {
|
||||
e.exportAnnotations(annotations);
|
||||
} catch (Throwable t) {
|
||||
logger.error("StatsExporter [{}] has thrown an exception:", t, e.name());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void exportShardStats() {
|
||||
logger.debug("Collecting shard stats");
|
||||
ShardStats[] shardStatsArray = indicesService.shardStats(CommonStatsFlags.ALL);
|
||||
|
||||
logger.debug("Exporting shards stats");
|
||||
for (StatsExporter e : exporters) {
|
||||
try {
|
||||
e.exportShardStats(shardStatsArray);
|
||||
} catch (Throwable t) {
|
||||
logger.error("StatsExporter [{}] has thrown an exception:", t, e.name());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void exportNodeStats() {
|
||||
logger.debug("Collecting node stats");
|
||||
NodeStats nodeStats = nodeService.stats();
|
||||
|
||||
logger.debug("Exporting node stats");
|
||||
for (StatsExporter e : exporters) {
|
||||
try {
|
||||
e.exportNodeStats(nodeStats);
|
||||
} catch (Throwable t) {
|
||||
logger.error("StatsExporter [{}] has thrown an exception:", t, e.name());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class IndicesLifeCycleListener extends IndicesLifecycle.Listener {
|
||||
@Override
|
||||
public void afterIndexShardStarted(IndexShard indexShard) {
|
||||
pendingAnnotationsQueue.add(new ShardEventAnnotation(System.currentTimeMillis(), ShardEventAnnotation.EventType.STARTED,
|
||||
indexShard.shardId(), indexShard.routingEntry()));
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void beforeIndexShardCreated(ShardId shardId) {
|
||||
pendingAnnotationsQueue.add(new ShardEventAnnotation(System.currentTimeMillis(), ShardEventAnnotation.EventType.CREATED,
|
||||
shardId, null));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard) {
|
||||
pendingAnnotationsQueue.add(new ShardEventAnnotation(System.currentTimeMillis(), ShardEventAnnotation.EventType.CLOSED,
|
||||
indexShard.shardId(), indexShard.routingEntry()));
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,63 @@
|
|||
package org.elasticsearch.marvel.monitor.annotation;
|
||||
/*
|
||||
* Licensed to ElasticSearch under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. ElasticSearch licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
|
||||
import org.elasticsearch.common.joda.Joda;
|
||||
import org.elasticsearch.common.joda.time.format.DateTimeFormatter;
|
||||
import org.elasticsearch.common.xcontent.ToXContent;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public abstract class Annotation {
|
||||
|
||||
public final static DateTimeFormatter datePrinter = Joda.forPattern("date_time").printer();
|
||||
|
||||
protected long timestamp;
|
||||
|
||||
public Annotation(long timestamp) {
|
||||
this.timestamp = timestamp;
|
||||
}
|
||||
|
||||
public long timestamp() {
|
||||
return timestamp;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return annotation's type as a short string without spaces
|
||||
*/
|
||||
public abstract String type();
|
||||
|
||||
/**
|
||||
* should return a short string based description of the annotation
|
||||
*/
|
||||
abstract String conciseDescription();
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "[" + type() + "] annotation: [" + conciseDescription() + "]";
|
||||
}
|
||||
|
||||
public XContentBuilder addXContentBody(XContentBuilder builder, ToXContent.Params params) throws IOException {
|
||||
builder.field("@timestamp", datePrinter.print(timestamp));
|
||||
builder.field("message", conciseDescription());
|
||||
return builder;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
package org.elasticsearch.marvel.monitor.annotation;
|
||||
/*
|
||||
* Licensed to ElasticSearch under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. ElasticSearch licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
|
||||
import org.elasticsearch.cluster.routing.ShardRouting;
|
||||
import org.elasticsearch.common.xcontent.ToXContent;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.index.shard.ShardId;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class ShardEventAnnotation extends Annotation {
|
||||
|
||||
private final ShardRouting shardRouting;
|
||||
private final ShardId shardId;
|
||||
private EventType event;
|
||||
|
||||
public enum EventType {
|
||||
CREATED,
|
||||
STARTED,
|
||||
CLOSED
|
||||
}
|
||||
|
||||
|
||||
public ShardEventAnnotation(long timestamp, EventType event, ShardId shardId, ShardRouting shardRouting) {
|
||||
super(timestamp);
|
||||
this.event = event;
|
||||
this.shardId = shardId;
|
||||
this.shardRouting = shardRouting;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
return "shard_event";
|
||||
}
|
||||
|
||||
@Override
|
||||
String conciseDescription() {
|
||||
return "[" + event + "]" + (shardRouting != null ? shardRouting : shardId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public XContentBuilder addXContentBody(XContentBuilder builder, ToXContent.Params params) throws IOException {
|
||||
super.addXContentBody(builder, params);
|
||||
builder.field("event", event);
|
||||
builder.field("index", shardId.index());
|
||||
builder.field("shard_id", shardId.id());
|
||||
if (shardRouting != null) {
|
||||
builder.field("routing");
|
||||
shardRouting.toXContent(builder, params);
|
||||
}
|
||||
|
||||
return builder;
|
||||
}
|
||||
}
|
||||
|
|
@ -29,10 +29,9 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
|
|||
import org.elasticsearch.cluster.routing.ShardRouting;
|
||||
import org.elasticsearch.common.collect.ImmutableMap;
|
||||
import org.elasticsearch.common.component.AbstractLifecycleComponent;
|
||||
import org.elasticsearch.common.joda.time.DateTimeZone;
|
||||
import org.elasticsearch.common.joda.Joda;
|
||||
import org.elasticsearch.common.joda.time.format.DateTimeFormat;
|
||||
import org.elasticsearch.common.joda.time.format.DateTimeFormatter;
|
||||
import org.elasticsearch.common.joda.time.format.ISODateTimeFormat;
|
||||
import org.elasticsearch.common.logging.ESLogger;
|
||||
import org.elasticsearch.common.logging.ESLoggerFactory;
|
||||
import org.elasticsearch.common.network.NetworkUtils;
|
||||
|
|
@ -45,6 +44,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
|
|||
import org.elasticsearch.common.xcontent.XContentType;
|
||||
import org.elasticsearch.common.xcontent.smile.SmileXContent;
|
||||
import org.elasticsearch.discovery.Discovery;
|
||||
import org.elasticsearch.marvel.monitor.annotation.Annotation;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
|
@ -64,9 +64,10 @@ public class ESExporter extends AbstractLifecycleComponent<ESExporter> implement
|
|||
final Discovery discovery;
|
||||
final String hostname;
|
||||
|
||||
// TODO: logger name is not good now. Figure out why.
|
||||
final ESLogger logger = ESLoggerFactory.getLogger(ESExporter.class.getName());
|
||||
|
||||
public final static DateTimeFormatter defaultDatePrinter = ISODateTimeFormat.dateTime().withZone(DateTimeZone.UTC);
|
||||
public final static DateTimeFormatter defaultDatePrinter = Joda.forPattern("date_time").printer();
|
||||
|
||||
boolean checkedForIndexTemplate = false;
|
||||
|
||||
|
|
@ -74,6 +75,7 @@ public class ESExporter extends AbstractLifecycleComponent<ESExporter> implement
|
|||
final ShardStatsRenderer shardStatsRenderer;
|
||||
final IndexStatsRenderer indexStatsRenderer;
|
||||
final IndicesStatsRenderer indicesStatsRenderer;
|
||||
final AnnotationsRenderer annotationsRenderer;
|
||||
|
||||
public ESExporter(Settings settings, Discovery discovery) {
|
||||
super(settings);
|
||||
|
|
@ -94,6 +96,7 @@ public class ESExporter extends AbstractLifecycleComponent<ESExporter> implement
|
|||
shardStatsRenderer = new ShardStatsRenderer();
|
||||
indexStatsRenderer = new IndexStatsRenderer();
|
||||
indicesStatsRenderer = new IndicesStatsRenderer();
|
||||
annotationsRenderer = new AnnotationsRenderer();
|
||||
|
||||
logger.info("ESExporter initialized. Targets: {}, index prefix [{}], index time format [{}]", hosts, indexPrefix, indexTimeFormat);
|
||||
}
|
||||
|
|
@ -107,13 +110,13 @@ public class ESExporter extends AbstractLifecycleComponent<ESExporter> implement
|
|||
@Override
|
||||
public void exportNodeStats(NodeStats nodeStats) {
|
||||
nodeStatsRenderer.reset(nodeStats);
|
||||
exportXContent("node_stats", nodeStatsRenderer);
|
||||
exportXContent(nodeStatsRenderer);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void exportShardStats(ShardStats[] shardStatsArray) {
|
||||
shardStatsRenderer.reset(shardStatsArray);
|
||||
exportXContent("shard_stats", shardStatsRenderer);
|
||||
exportXContent(shardStatsRenderer);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -127,8 +130,8 @@ public class ESExporter extends AbstractLifecycleComponent<ESExporter> implement
|
|||
return;
|
||||
}
|
||||
try {
|
||||
addXContentRendererToConnection(conn, "index_stats", indexStatsRenderer);
|
||||
addXContentRendererToConnection(conn, "indices_stats", indicesStatsRenderer);
|
||||
addXContentRendererToConnection(conn, indexStatsRenderer);
|
||||
addXContentRendererToConnection(conn, indicesStatsRenderer);
|
||||
sendCloseExportingConnection(conn);
|
||||
} catch (IOException e) {
|
||||
logger.error("error sending data", e);
|
||||
|
|
@ -136,6 +139,13 @@ public class ESExporter extends AbstractLifecycleComponent<ESExporter> implement
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void exportAnnotations(Annotation[] annotations) {
|
||||
annotationsRenderer.reset(annotations);
|
||||
exportXContent(annotationsRenderer);
|
||||
}
|
||||
|
||||
|
||||
private HttpURLConnection openExportingConnection() {
|
||||
if (!checkedForIndexTemplate) {
|
||||
if (!checkForIndexTemplate()) {
|
||||
|
|
@ -152,14 +162,14 @@ public class ESExporter extends AbstractLifecycleComponent<ESExporter> implement
|
|||
return conn;
|
||||
}
|
||||
|
||||
private void addXContentRendererToConnection(HttpURLConnection conn, String type,
|
||||
private void addXContentRendererToConnection(HttpURLConnection conn,
|
||||
MultiXContentRenderer renderer) throws IOException {
|
||||
OutputStream os = conn.getOutputStream();
|
||||
// TODO: find a way to disable builder's substream flushing or something neat solution
|
||||
for (int i = 0; i < renderer.length(); i++) {
|
||||
XContentBuilder builder = XContentFactory.smileBuilder(os);
|
||||
builder.startObject().startObject("index")
|
||||
.field("_index", getIndexName()).field("_type", type).endObject().endObject();
|
||||
.field("_index", getIndexName()).field("_type", renderer.type(i)).endObject().endObject();
|
||||
builder.flush();
|
||||
os.write(SmileXContent.smileXContent.streamSeparator());
|
||||
|
||||
|
|
@ -183,18 +193,17 @@ public class ESExporter extends AbstractLifecycleComponent<ESExporter> implement
|
|||
}
|
||||
}
|
||||
|
||||
private void exportXContent(String type, MultiXContentRenderer xContentRenderer) {
|
||||
private void exportXContent(MultiXContentRenderer xContentRenderer) {
|
||||
if (xContentRenderer.length() == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
logger.debug("exporting {}", type);
|
||||
HttpURLConnection conn = openExportingConnection();
|
||||
if (conn == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
addXContentRendererToConnection(conn, type, xContentRenderer);
|
||||
addXContentRendererToConnection(conn, xContentRenderer);
|
||||
sendCloseExportingConnection(conn);
|
||||
} catch (IOException e) {
|
||||
logger.error("error sending data", e);
|
||||
|
|
@ -315,6 +324,8 @@ public class ESExporter extends AbstractLifecycleComponent<ESExporter> implement
|
|||
|
||||
int length();
|
||||
|
||||
String type(int i);
|
||||
|
||||
void render(int index, XContentBuilder builder) throws IOException;
|
||||
}
|
||||
|
||||
|
|
@ -368,6 +379,11 @@ public class ESExporter extends AbstractLifecycleComponent<ESExporter> implement
|
|||
return 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String type(int i) {
|
||||
return "node_stats";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void render(int index, XContentBuilder builder) throws IOException {
|
||||
builder.startObject();
|
||||
|
|
@ -394,6 +410,11 @@ public class ESExporter extends AbstractLifecycleComponent<ESExporter> implement
|
|||
return stats == null ? 0 : stats.length;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String type(int i) {
|
||||
return "shard_stats";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void render(int index, XContentBuilder builder) throws IOException {
|
||||
builder.startObject();
|
||||
|
|
@ -425,6 +446,11 @@ public class ESExporter extends AbstractLifecycleComponent<ESExporter> implement
|
|||
return stats == null ? 0 : stats.length;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String type(int i) {
|
||||
return "index_stats";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void render(int index, XContentBuilder builder) throws IOException {
|
||||
builder.startObject();
|
||||
|
|
@ -460,6 +486,11 @@ public class ESExporter extends AbstractLifecycleComponent<ESExporter> implement
|
|||
return totalStats == null ? 0 : 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String type(int i) {
|
||||
return "indices_stats";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void render(int index, XContentBuilder builder) throws IOException {
|
||||
assert index == 0;
|
||||
|
|
@ -476,5 +507,35 @@ public class ESExporter extends AbstractLifecycleComponent<ESExporter> implement
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
class AnnotationsRenderer implements MultiXContentRenderer {
|
||||
|
||||
Annotation[] annotations;
|
||||
ToXContent.Params xContentParams = ToXContent.EMPTY_PARAMS;
|
||||
|
||||
public void reset(Annotation[] annotations) {
|
||||
this.annotations = annotations;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int length() {
|
||||
return annotations == null ? 0 : annotations.length;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String type(int i) {
|
||||
return annotations[i].type();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void render(int index, XContentBuilder builder) throws IOException {
|
||||
builder.startObject();
|
||||
addNodeInfo(builder, "node");
|
||||
annotations[index].addXContentBody(builder, xContentParams);
|
||||
builder.endObject();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ import org.elasticsearch.action.admin.indices.stats.IndexStats;
|
|||
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
|
||||
import org.elasticsearch.action.admin.indices.stats.ShardStats;
|
||||
import org.elasticsearch.common.component.LifecycleComponent;
|
||||
import org.elasticsearch.marvel.monitor.annotation.Annotation;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
|
|
@ -38,4 +39,5 @@ public interface StatsExporter<T> extends LifecycleComponent<T> {
|
|||
|
||||
void exportIndicesStats(IndicesStatsResponse indicesStats);
|
||||
|
||||
void exportAnnotations(Annotation[] annotations);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue